ingestr 0.12.4__tar.gz → 0.12.5__tar.gz

{ingestr-0.12.4 → ingestr-0.12.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.12.4
+Version: 0.12.5
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -17,11 +17,11 @@ Requires-Python: >=3.9
 Requires-Dist: asana==3.2.3
 Requires-Dist: confluent-kafka>=2.6.1
 Requires-Dist: databricks-sql-connector==2.9.3
-Requires-Dist: dlt==1.4.0
+Requires-Dist: dlt==1.5.0
 Requires-Dist: duckdb-engine==0.13.5
 Requires-Dist: duckdb==1.1.3
 Requires-Dist: facebook-business==20.0.0
-Requires-Dist: google-analytics-data==0.18.15
+Requires-Dist: google-analytics-data==0.18.16
 Requires-Dist: google-api-python-client==2.130.0
 Requires-Dist: google-cloud-bigquery-storage==2.24.0
 Requires-Dist: mysql-connector-python==9.1.0
@@ -34,7 +34,7 @@ Requires-Dist: pyathena==3.9.0
 Requires-Dist: pymongo==4.10.1
 Requires-Dist: pymysql==1.1.1
 Requires-Dist: pyrate-limiter==3.7.0
-Requires-Dist: redshift-connector==2.1.3
+Requires-Dist: redshift-connector==2.1.5
 Requires-Dist: rich==13.9.4
 Requires-Dist: rudder-sdk-python==2.1.4
 Requires-Dist: s3fs==2024.10.0

{ingestr-0.12.4 → ingestr-0.12.5}/ingestr/main.py

@@ -57,8 +57,9 @@ class SpinnerCollector(Collector):
         name: str,
         inc: int = 1,
         total: Optional[int] = None,
-        message: Optional[str] = None,
+        message: Optional[str] = None,  # type: ignore
         label: str = "",
+        **kwargs,
     ) -> None:
         self.status.update(self.current_step)
 

{ingestr-0.12.4 → ingestr-0.12.5}/ingestr/src/arrow/__init__.py

@@ -1,7 +1,3 @@
-"""Source that loads tables form Airtable.
-Supports whitelisting of tables or loading of all tables from a specified base.
-"""
-
 from typing import Any, Optional
 
 import dlt

{ingestr-0.12.4 → ingestr-0.12.5}/ingestr/src/sources.py

@@ -18,7 +18,6 @@ from urllib.parse import ParseResult, parse_qs, quote, urlparse
 
 import dlt
 import pendulum
-import sqlalchemy
 from dlt.common.configuration.specs import (
     AwsCredentials,
 )
@@ -42,7 +41,6 @@ from dlt.sources.sql_database.schema_types import (
 )
 from sqlalchemy import Column
 from sqlalchemy import types as sa
-from sqlalchemy.dialects import mysql
 
 from ingestr.src.adjust import REQUIRED_CUSTOM_DIMENSIONS, adjust_source
 from ingestr.src.adjust.adjust_helpers import parse_filters
@@ -67,6 +65,12 @@ from ingestr.src.mongodb import mongodb_collection
 from ingestr.src.notion import notion_databases
 from ingestr.src.shopify import shopify_source
 from ingestr.src.slack import slack_source
+from ingestr.src.sql_database.callbacks import (
+    chained_query_adapter_callback,
+    custom_query_variable_subsitution,
+    limit_callback,
+    type_adapter_callback,
+)
 from ingestr.src.stripe_analytics import stripe_source
 from ingestr.src.table_definition import TableDefinition, table_string_to_dataclass
 from ingestr.src.tiktok_ads import tiktok_source
@@ -103,21 +107,18 @@ class SqlSource:
                 kwargs.get("incremental_key", ""),
                 initial_value=start_value,
                 end_value=end_value,
+                range_end="closed",
+                range_start="closed",
             )
 
         if uri.startswith("mysql://"):
             uri = uri.replace("mysql://", "mysql+pymysql://")
 
-        reflection_level = kwargs.get("sql_reflection_level")
-
-        query_adapter_callback = None
+        query_adapters = []
         if kwargs.get("sql_limit"):
-
-            def query_adapter_callback(query, table):
-                query = query.limit(kwargs.get("sql_limit"))
-                if kwargs.get("incremental_key"):
-                    query = query.order_by(kwargs.get("incremental_key"))
-                return query
+            query_adapters.append(
+                limit_callback(kwargs.get("sql_limit"), kwargs.get("incremental_key"))
+            )
 
         defer_table_reflect = False
         sql_backend = kwargs.get("sql_backend", "sqlalchemy")
@@ -196,38 +197,10 @@ class SqlSource:
                     if getattr(engine, "may_dispose_after_use", False):
                         engine.dispose()
 
-            dlt.sources.sql_database.table_rows = table_rows
-
-            def query_adapter_callback(query, table, incremental=None, engine=None):
-                params = {}
-                if incremental:
-                    params["interval_start"] = (
-                        incremental.last_value
-                        if incremental.last_value is not None
-                        else datetime(year=1, month=1, day=1)
-                    )
-                    if incremental.end_value is not None:
-                        params["interval_end"] = incremental.end_value
-                else:
-                    if ":interval_start" in query_value:
-                        params["interval_start"] = (
-                            datetime.min
-                            if kwargs.get("interval_start") is None
-                            else kwargs.get("interval_start")
-                        )
-                    if ":interval_end" in query_value:
-                        params["interval_end"] = (
-                            datetime.max
-                            if kwargs.get("interval_end") is None
-                            else kwargs.get("interval_end")
-                        )
-
-                return sqlalchemy.text(query_value).bindparams(**params)
+            dlt.sources.sql_database.table_rows = table_rows  # type: ignore
 
-        def type_adapter_callback(sql_type):
-            if isinstance(sql_type, mysql.SET):
-                return sa.JSON
-            return sql_type
+            # override the query adapters, the only one we want is the one here in the case of custom queries
+            query_adapters = [custom_query_variable_subsitution(query_value, kwargs)]
 
         builder_res = self.table_builder(
             credentials=ConnectionStringCredentials(uri),
@@ -236,8 +209,8 @@ class SqlSource:
             incremental=incremental,
             backend=sql_backend,
             chunk_size=kwargs.get("page_size", None),
-            reflection_level=reflection_level,
-            query_adapter_callback=query_adapter_callback,
+            reflection_level=kwargs.get("sql_reflection_level", None),
+            query_adapter_callback=chained_query_adapter_callback(query_adapters),
             type_adapter_callback=type_adapter_callback,
             table_adapter_callback=table_adapter_exclude_columns(
                 kwargs.get("sql_exclude_columns", [])

ingestr-0.12.5/ingestr/src/sql_database/callbacks.py

@@ -0,0 +1,66 @@
+from datetime import datetime
+
+from sqlalchemy import text
+from sqlalchemy import types as sa
+from sqlalchemy.dialects import mysql
+
+
+def type_adapter_callback(sql_type):
+    if isinstance(sql_type, mysql.SET):
+        return sa.JSON
+    return sql_type
+
+
+def chained_query_adapter_callback(query_adapters):
+    """
+    This function is used to chain multiple query adapters together,.
+    This gives us the flexibility to introduce various adapters based on the given command parameters.
+    """
+
+    def callback(query, table):
+        for adapter in query_adapters:
+            query = adapter(query, table)
+
+        return query
+
+    return callback
+
+
+def limit_callback(sql_limit: int, incremental_key: str):
+    def callback(query, table):
+        query = query.limit(sql_limit)
+        if incremental_key:
+            query = query.order_by(incremental_key)
+        return query
+
+    return callback
+
+
+def custom_query_variable_subsitution(query_value: str, kwargs: dict):
+    def callback(query, table, incremental=None, engine=None):
+        params = {}
+        if incremental:
+            params["interval_start"] = (
+                incremental.last_value
+                if incremental.last_value is not None
+                else datetime(year=1, month=1, day=1)
+            )
+            if incremental.end_value is not None:
+                params["interval_end"] = incremental.end_value
+        else:
+            if ":interval_start" in query_value:
+                params["interval_start"] = (
+                    datetime.min
+                    if kwargs.get("interval_start") is None
+                    else kwargs.get("interval_start")
+                )
+            if ":interval_end" in query_value:
+                params["interval_end"] = (
+                    datetime.max
+                    if kwargs.get("interval_end") is None
+                    else kwargs.get("interval_end")
+                )
+
+        return text(query_value).bindparams(**params)
+
+    return callback

ingestr-0.12.5/ingestr/src/version.py

@@ -0,0 +1 @@
+__version__ = "0.12.5"

{ingestr-0.12.4 → ingestr-0.12.5}/requirements.txt

@@ -1,6 +1,6 @@
 confluent-kafka>=2.6.1
 databricks-sql-connector==2.9.3
-dlt==1.4.0
+dlt==1.5.0
 duckdb_engine==0.13.5
 duckdb==1.1.3
 google-api-python-client==2.130.0
@@ -12,7 +12,7 @@ py-machineid==0.6.0
 pymongo==4.10.1
 pymysql==1.1.1
 pyrate-limiter==3.7.0
-redshift-connector==2.1.3
+redshift-connector==2.1.5
 rich==13.9.4
 rudder-sdk-python==2.1.4
 snowflake-sqlalchemy==1.6.1
@@ -30,5 +30,5 @@ types-requests==2.32.0.20240907
 s3fs==2024.10.0
 pyarrow==18.1.0
 pyathena==3.9.0
-google-analytics-data==0.18.15
+google-analytics-data==0.18.16
 asana==3.2.3

ingestr-0.12.4/ingestr/src/version.py

@@ -1 +0,0 @@
-__version__ = "0.12.4"