ingestr 0.13.51__tar.gz → 0.13.52__tar.gz

{ingestr-0.13.51 → ingestr-0.13.52}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.51
+Version: 0.13.52
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues

{ingestr-0.13.51 → ingestr-0.13.52}/docs/supported-sources/solidgate.md

@@ -39,8 +39,9 @@ The result of this command will be a table in the `Solidgate.duckdb` database wi
 
 Solidgate source allows ingesting the following sources into separate tables:
 - `subscriptions`: Provides a comprehensive view of customer subscriptions, including subscription IDs, statuses, and key timestamps such as creation, update, and expiration dates.
-- `apm-orders`: Provides essential information for anti-fraud purposes, including order IDs, transaction statuses, amounts, currencies, and payment methods, along with crucial customer details such as email addresses.
-- `card-orders`: Provides detailed information on orders processed via card payments, including transaction data, payment status, and customer details.
+- `apm_orders`: Provides essential information for anti-fraud purposes, including order IDs, transaction statuses, amounts, currencies, and payment methods, along with crucial customer details such as email addresses.
+- `card_orders`: Provides detailed information on orders processed via card payments, including transaction data, payment status, and customer details.
+- `financial_entries`: Provides a detailed transaction-level breakdown of financial records.
 
 Use these as `--source-table` parameter in the `ingestr ingest` command.
 

{ingestr-0.13.51 → ingestr-0.13.52}/docs/supported-sources/sqlite.md

@@ -1,7 +1,7 @@
 # SQLite
 SQLite is a C-language library that implements a small, fast, self-contained, high-reliability, full-featured, SQL database engine.
 
-ingestr supports SQLite as a source.
+ingestr supports SQLite as a source and a destination.
 
 ## URI format
 The URI format for SQLite is as follows:

{ingestr-0.13.51 → ingestr-0.13.52}/ingestr/main.py

@@ -58,6 +58,7 @@ class LoaderFileFormat(str, Enum):
 
 
 class SqlBackend(str, Enum):
+    default = "default"
     sqlalchemy = "sqlalchemy"
     pyarrow = "pyarrow"
     connectorx = "connectorx"
@@ -187,7 +188,7 @@ def ingest(
             help="The SQL backend to use",
             envvar=["SQL_BACKEND", "INGESTR_SQL_BACKEND"],
         ),
-    ] = SqlBackend.pyarrow,  # type: ignore
+    ] = SqlBackend.default,  # type: ignore
     loader_file_format: Annotated[
         Optional[LoaderFileFormat],
         typer.Option(
@@ -289,7 +290,11 @@ def ingest(
     from ingestr.src.collector.spinner import SpinnerCollector
     from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
-    from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
+    from ingestr.src.filters import (
+        cast_set_to_list,
+        cast_spanner_types,
+        handle_mysql_empty_dates,
+    )
     from ingestr.src.sources import MongoDbSource
 
     def report_errors(run_info: LoadInfo):
@@ -517,6 +522,15 @@ def ingest(
             if interval_end:
                 interval_end = interval_end.date()  # type: ignore
 
+        if factory.source_scheme.startswith("spanner"):
+            # we tend to use the 'pyarrow' backend in general, however, it has issues with JSON objects, so we override it to 'sqlalchemy' for Spanner.
+            if sql_backend.value == SqlBackend.default:
+                sql_backend = SqlBackend.sqlalchemy
+
+        # this allows us to identify the cases where the user does not have a preference, so that for some sources we can override it.
+        if sql_backend == SqlBackend.default:
+            sql_backend = SqlBackend.pyarrow
+
         dlt_source = source.dlt_source(
             uri=source_uri,
             table=source_table,
@@ -535,6 +549,9 @@ def ingest(
         if factory.source_scheme.startswith("mysql"):
             resource.for_each(dlt_source, lambda x: x.add_map(handle_mysql_empty_dates))
 
+        if factory.source_scheme.startswith("spanner"):
+            resource.for_each(dlt_source, lambda x: x.add_map(cast_spanner_types))
+
         if yield_limit:
             resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
 

ingestr-0.13.52/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.52"

{ingestr-0.13.51 → ingestr-0.13.52}/ingestr/src/destinations.py

@@ -468,3 +468,15 @@ class S3Destination:
 
     def post_load(self) -> None:
         pass
+
+
+class SqliteDestination(GenericSqlDestination):
+    def dlt_dest(self, uri: str, **kwargs):
+        return dlt.destinations.sqlalchemy(credentials=uri)
+
+    def dlt_run_params(self, uri: str, table: str, **kwargs):
+        return {
+            #https://dlthub.com/docs/dlt-ecosystem/destinations/sqlalchemy#dataset-files
+            "dataset_name": "main",
+            "table_name": table,
+        }

{ingestr-0.13.51 → ingestr-0.13.52}/ingestr/src/factory.py

@@ -15,6 +15,7 @@ from ingestr.src.destinations import (
     RedshiftDestination,
     S3Destination,
     SnowflakeDestination,
+    SqliteDestination,
     SynapseDestination,
 )
 from ingestr.src.sources import (
@@ -54,12 +55,12 @@ from ingestr.src.sources import (
     SalesforceSource,
     ShopifySource,
     SlackSource,
+    SmartsheetSource,
     SolidgateSource,
     SqlSource,
     StripeAnalyticsSource,
     TikTokSource,
     ZendeskSource,
-    SmartsheetSource,
 )
 
 SQL_SOURCE_SCHEMES = [
@@ -182,6 +183,7 @@ class SourceDestinationFactory:
         "clickhouse+native": ClickhouseDestination,
         "clickhouse": ClickhouseDestination,
         "s3": S3Destination,
+        "sqlite": SqliteDestination,
     }
 
     def __init__(self, source_uri: str, destination_uri: str):

{ingestr-0.13.51 → ingestr-0.13.52}/ingestr/src/filters.py

@@ -7,6 +7,20 @@ def cast_set_to_list(row):
     return row
 
 
+def cast_spanner_types(row):
+    if not isinstance(row, dict):
+        return row
+
+    from google.cloud.spanner_v1.data_types import JsonObject
+
+    for key in row.keys():
+        if isinstance(row[key], JsonObject):
+            import json
+
+            row[key] = json.loads(row[key].serialize())
+    return row
+
+
 def handle_mysql_empty_dates(row):
     # MySQL returns empty dates as 0000-00-00, which is not a valid date, we handle them here.
     if not isinstance(row, dict):

{ingestr-0.13.51 → ingestr-0.13.52}/ingestr/src/solidgate/__init__.py

@@ -43,7 +43,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="apm-orders",
+        name="apm_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -69,7 +69,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="card-orders",
+        name="card_orders",
         write_disposition="merge",
         primary_key="order_id",
         columns={
@@ -95,7 +95,7 @@ def solidgate_source(
         yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
 
     @dlt.resource(
-        name="financial-entries",
+        name="financial_entries",
         write_disposition="merge",
         primary_key="id",
         columns={

ingestr-0.13.51/ingestr/src/buildinfo.py

@@ -1 +0,0 @@
-version = "v0.13.51"