ingestr 0.13.48__tar.gz → 0.13.50__tar.gz

{ingestr-0.13.48 → ingestr-0.13.50}/.gitleaksignore

@@ -1,4 +1,5 @@
 ingestr/src/telemetry/event.py:generic-api-key:17
 ingestr/src/testdata/fakebqcredentials.json:private-key:5
 docs/supported-sources/shopify.md:generic-api-key:26
+docs/supported-sources/smartsheets.md:generic-api-key:38
 ingestr/src/google_analytics/helpers_test.py:generic-api-key:36

{ingestr-0.13.48 → ingestr-0.13.50}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: ingestr
-Version: 0.13.48
+Version: 0.13.50
 Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
 Project-URL: Homepage, https://github.com/bruin-data/ingestr
 Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -163,6 +163,7 @@ Requires-Dist: shellingham==1.5.4
 Requires-Dist: simple-salesforce==1.12.6
 Requires-Dist: simplejson==3.20.1
 Requires-Dist: six==1.17.0
+Requires-Dist: smartsheet-python-sdk==3.0.5
 Requires-Dist: smmap==5.0.2
 Requires-Dist: snowflake-connector-python==3.14.0
 Requires-Dist: snowflake-sqlalchemy==1.6.1
@@ -467,6 +468,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Smartsheet</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Stripe</td>
         <td>✅</td>

{ingestr-0.13.48 → ingestr-0.13.50}/README.md

@@ -263,6 +263,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
         <td>✅</td>
         <td>-</td>
     </tr>
+    <tr>
+        <td>Smartsheet</td>
+        <td>✅</td>
+        <td>-</td>
+    </tr>
     <tr>
         <td>Stripe</td>
         <td>✅</td>

{ingestr-0.13.48 → ingestr-0.13.50}/docs/.vitepress/config.mjs

@@ -79,6 +79,7 @@ export default defineConfig({
                 text: "Google BigQuery",
                 link: "/supported-sources/bigquery.md",
               },
+              { text: "GCP Spanner", link: "/supported-sources/spanner.md" },
               { text: "IBM Db2", link: "/supported-sources/db2.md" },
               { text: "Kafka", link: "/supported-sources/kafka.md" },
               { text: "Local CSV Files", link: "/supported-sources/csv.md" },
@@ -139,7 +140,7 @@ export default defineConfig({
               { text: "Salesforce", link: "/supported-sources/salesforce.md" },
               { text: "Shopify", link: "/supported-sources/shopify.md" },
               { text: "Slack", link: "/supported-sources/slack.md" },
-              { text: "Spanner", link: "/supported-sources/spanner.md" },
+              { text: "Smartsheet", link: "/supported-sources/smartsheets.md" },
               { text: "Solidgate", link: "/supported-sources/solidgate.md" },
               { text: "Stripe", link: "/supported-sources/stripe.md" },
               { text: "TikTok Ads", link: "/supported-sources/tiktok-ads.md" },

ingestr-0.13.50/docs/supported-sources/smartsheets.md

@@ -0,0 +1,47 @@
+# Smartsheet
+
+[Smartsheet](https://www.smartsheet.com/) is a software as a service (SaaS) offering for collaboration and work management.
+
+ingestr supports Smartsheet as a source.
+
+## URI format
+
+The URI format for Smartsheet is as follows:
+
+```plaintext
+smartsheet://?access_token=<access_token>
+```
+
+URI parameters:
+
+- `access_token`: Your Smartsheet API access token.
+
+The URI is used to connect to the Smartsheet API for extracting data. You can generate an access token in Smartsheet by navigating to Account > Personal Settings > API Access.
+
+## Setting up a Smartsheet Integration
+
+To set up a Smartsheet integration, you'll need an API Access Token.
+
+1. Log in to Smartsheet.
+2. Click on "Account" in the bottom left corner, then "Personal Settings".
+3. Go to the "API Access" tab.
+4. Click "Generate new access token".
+5. Give your token a name and click "OK".
+6. Copy the generated token. This will be your `access_token`.
+
+The source table you'll use for ingestr will be the `sheet_id` of the Smartsheet you want to ingest. You can find the `sheet_id` by opening the sheet in Smartsheet and going to File > Properties. The Sheet ID will be listed there.
+
+Let's say your access token is `llk2k3j4l5k6j7h8g9f0` and the sheet ID is `1234567890123456`, here's a sample command that will copy the data from Smartsheet into a DuckDB database:
+
+```sh
+ingestr ingest \
+    --source-uri 'smartsheet://?access_token=llk2k3j4l5k6j7h8g9f0' \
+    --source-table '1234567890123456' \
+    --dest-uri 'duckdb:///smartsheet_data.duckdb' \
+    --dest-table 'des.my_sheet_data'
+```
+
+The result of this command will be a `my_sheet_data` table containing data from your Smartsheet in the `smartsheet_data.duckdb` database.
+
+> [!CAUTION]
+> Smartsheet integration does not currently support incremental loading. Every time you run the command, the entire sheet will be copied from Smartsheet to the destination. This can be slow for large sheets. 

{ingestr-0.13.48 → ingestr-0.13.50}/docs/supported-sources/spanner.md

@@ -1,7 +1,7 @@
-# Spanner
-[Spanner](https://cloud.google.com/spanner) is a fully managed, scalable, and strongly consistent database service.
+# GCP Spanner
+[GCP Spanner](https://cloud.google.com/spanner) is a fully managed, scalable, and strongly consistent database service.
 
-ingestr supports Spanner as a source.
+ingestr supports GCP Spanner as a source.
 
 ## URI format
 The URI format for Spanner is as follows:
@@ -33,7 +33,7 @@ ingestr ingest \
 
 ```
 
-<img alt="pipedrive_img" src="../media/spanner_ingestion.png"/>
+<img alt="GCP Spanner Ingestion Example" src="../media/gcp_spanner_ingestion.png"/>
 
 > [!NOTE]
 > Currently, only GoogleSQL databases are supported.

{ingestr-0.13.48 → ingestr-0.13.50}/ingestr/main.py

@@ -290,6 +290,7 @@ def ingest(
     from ingestr.src.destinations import AthenaDestination
     from ingestr.src.factory import SourceDestinationFactory
     from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
+    from ingestr.src.sources import MongoDbSource
 
     def report_errors(run_info: LoadInfo):
         for load_package in run_info.load_packages:
@@ -537,6 +538,13 @@ def ingest(
         if yield_limit:
             resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
 
+        if isinstance(source, MongoDbSource):
+            from ingestr.src.resource import TypeHintMap
+
+            resource.for_each(
+                dlt_source, lambda x: x.add_map(TypeHintMap().type_hint_map)
+            )
+
         def col_h(x):
             if column_hints:
                 x.apply_hints(columns=column_hints)

ingestr-0.13.50/ingestr/src/buildinfo.py

@@ -0,0 +1 @@
+version = "v0.13.50"

{ingestr-0.13.48 → ingestr-0.13.50}/ingestr/src/mongodb/__init__.py

@@ -14,7 +14,7 @@ from .helpers import (
 )
 
 
-@dlt.source
+@dlt.source(max_table_nesting=0)
 def mongodb(
     connection_url: str = dlt.secrets.value,
     database: Optional[str] = dlt.config.value,
@@ -75,6 +75,7 @@ def mongodb(
             primary_key="_id",
             write_disposition=write_disposition,
             spec=MongoDbCollectionConfiguration,
+            max_table_nesting=0,
         )(
             client,
             collection,

ingestr-0.13.50/ingestr/src/resource.py

@@ -0,0 +1,40 @@
+from typing import Callable
+
+from dlt.sources import DltResource, DltSource
+
+
+def for_each(
+    source: DltSource | DltResource, ex: Callable[[DltResource], None | DltResource]
+):
+    """
+    Apply a function to each resource in a source.
+    """
+    if hasattr(source, "selected_resources") and source.selected_resources:
+        resource_names = list(source.selected_resources.keys())
+        for res in resource_names:
+            ex(source.resources[res])  # type: ignore[union-attr]
+    else:
+        ex(source)  # type: ignore[arg-type]
+
+
+class TypeHintMap:
+    def __init__(self):
+        self.handled_typehints = False
+
+    def type_hint_map(self, item):
+        if self.handled_typehints:
+            return item
+
+        array_cols = []
+        for col in item:
+            if isinstance(item[col], (list, tuple)):
+                array_cols.append(col)
+        if array_cols:
+            import dlt
+
+            source = dlt.current.source()
+            columns = [{"name": col, "data_type": "json"} for col in array_cols]
+            for_each(source, lambda x: x.apply_hints(columns=columns))
+
+        self.handled_typehints = True
+        return item

ingestr-0.13.50/ingestr/src/smartsheets/__init__.py

@@ -0,0 +1,54 @@
+from typing import Iterable
+
+import dlt
+import smartsheet  # type: ignore
+from dlt.extract import DltResource
+
+
+@dlt.source
+def smartsheet_source(
+    access_token: str,
+    sheet_id: str,
+) -> Iterable[DltResource]:
+    """
+    A DLT source for Smartsheet.
+
+    Args:
+        access_token: The Smartsheet API access token.
+        sheet_id: The ID of the sheet to load.
+
+    Returns:
+        An iterable of DLT resources.
+    """
+
+    # Initialize Smartsheet client
+    smartsheet_client = smartsheet.Smartsheet(access_token)
+    smartsheet_client.errors_as_exceptions(True)
+
+    # The SDK expects sheet_id to be an int
+    sheet_id_int = int(sheet_id)
+    # Sanitize the sheet name to be a valid resource name
+    # We get objectValue to ensure `name` attribute is populated for the sheet
+    sheet_details = smartsheet_client.Sheets.get_sheet(
+        sheet_id_int, include=["objectValue"]
+    )
+    sheet_name = sheet_details.name
+    resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
+
+    yield dlt.resource(
+        _get_sheet_data(smartsheet_client, sheet_id_int),
+        name=resource_name,
+        write_disposition="replace",
+    )
+
+
+def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
+    """Helper function to get all rows from a sheet."""
+    sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
+    # Transform rows to a list of dictionaries
+    column_titles = [col.title for col in sheet.columns]
+    for row in sheet.rows:
+        row_data = {}
+        for i, cell in enumerate(row.cells):
+            row_data[column_titles[i]] = cell.value
+        yield row_data

{ingestr-0.13.48 → ingestr-0.13.50}/ingestr/src/solidgate/helpers.py

@@ -50,18 +50,14 @@ class SolidgateClient:
                 data = response_json["subscriptions"]
                 for _, value in data.items():
                     if "updated_at" in value:
-                        value["updated_at"] = pendulum.parse(
-                            value["updated_at"]
-                        )
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
                     yield value
 
             else:
                 data = response_json["orders"]
                 for value in data:
                     if "updated_at" in value:
-                        value["updated_at"] = pendulum.parse(
-                        value["updated_at"]
-                    )
+                        value["updated_at"] = pendulum.parse(value["updated_at"])
                     yield value
 
             next_page_iterator = response_json.get("metadata", {}).get(

{ingestr-0.13.48 → ingestr-0.13.50}/ingestr/src/sources.py

@@ -388,6 +388,7 @@ class MongoDbSource:
             parallel=True,
             incremental=incremental,
         )
+        table_instance.max_table_nesting = 1
 
         return table_instance
 
@@ -697,46 +698,47 @@ class StripeAnalyticsSource:
         if not api_key:
             raise ValueError("api_key in the URI is required to connect to Stripe")
 
-        endpoint = None
-        if table == "balancetransaction":
-            table = "BalanceTransaction"
-        else:
-            table = table.capitalize()
-
-        if table in [
-            "Subscription",
-            "Account",
-            "Coupon",
-            "Customer",
-            "Product",
-            "Price",
-            "BalanceTransaction",
-            "Invoice",
-            "Event",
-            "Charge",
-        ]:
-            endpoint = table
-        else:
-            raise ValueError(
-                f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
-            )
-
-        date_args = {}
-        if kwargs.get("interval_start"):
-            date_args["start_date"] = kwargs.get("interval_start")
-
-        if kwargs.get("interval_end"):
-            date_args["end_date"] = kwargs.get("interval_end")
+        table = table.lower()
 
-        from ingestr.src.stripe_analytics import stripe_source
+        from ingestr.src.stripe_analytics.settings import (
+            ENDPOINTS,
+            INCREMENTAL_ENDPOINTS,
+        )
 
-        return stripe_source(
-            endpoints=[
-                endpoint,
-            ],
-            stripe_secret_key=api_key[0],
-            **date_args,
-        ).with_resources(endpoint)
+        if table in ENDPOINTS:
+            endpoint = ENDPOINTS[table]
+            from ingestr.src.stripe_analytics import stripe_source
+
+            return stripe_source(
+                endpoints=[
+                    endpoint,
+                ],
+                stripe_secret_key=api_key[0],
+                start_date=kwargs.get("interval_start", None),
+                end_date=kwargs.get("interval_end", None),
+            ).with_resources(endpoint)
+
+        elif table in INCREMENTAL_ENDPOINTS:
+            endpoint = INCREMENTAL_ENDPOINTS[table]
+            from ingestr.src.stripe_analytics import incremental_stripe_source
+
+            def nullable_date(date_str: Optional[str]):
+                if date_str:
+                    return ensure_pendulum_datetime(date_str)
+                return None
+
+            return incremental_stripe_source(
+                endpoints=[
+                    endpoint,
+                ],
+                stripe_secret_key=api_key[0],
+                initial_start_date=nullable_date(kwargs.get("interval_start", None)),
+                end_date=nullable_date(kwargs.get("interval_end", None)),
+            ).with_resources(endpoint)
+
+        raise ValueError(
+            f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
+        )
 
 
 class FacebookAdsSource:
@@ -2433,6 +2435,37 @@ class AttioSource:
             raise UnsupportedResourceError(table_name, "Attio")
 
 
+class SmartsheetSource:
+    def handles_incrementality(self) -> bool:
+        return False
+
+    # smartsheet://?access_token=<access_token>
+    def dlt_source(self, uri: str, table: str, **kwargs):
+        if kwargs.get("incremental_key"):
+            raise ValueError("Incremental loads are not supported for Smartsheet")
+
+        if not table:
+            raise ValueError(
+                "Source table (sheet_id) is required to connect to Smartsheet"
+            )
+
+        source_parts = urlparse(uri)
+        source_fields = parse_qs(source_parts.query)
+        access_token = source_fields.get("access_token")
+
+        if not access_token:
+            raise ValueError(
+                "access_token in the URI is required to connect to Smartsheet"
+            )
+
+        from ingestr.src.smartsheets import smartsheet_source
+
+        return smartsheet_source(
+            access_token=access_token[0],
+            sheet_id=table,  # table is now a single sheet_id
+        )
+
+
 class SolidgateSource:
     def handles_incrementality(self) -> bool:
         return True

{ingestr-0.13.48 → ingestr-0.13.50}/ingestr/src/stripe_analytics/__init__.py

@@ -8,12 +8,11 @@ from dlt.sources import DltResource
 from pendulum import DateTime
 
 from .helpers import pagination, transform_date
-from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
 
 
 @dlt.source(max_table_nesting=0)
 def stripe_source(
-    endpoints: Tuple[str, ...] = ENDPOINTS,
+    endpoints: Tuple[str, ...],
     stripe_secret_key: str = dlt.secrets.value,
     start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,
@@ -53,7 +52,7 @@ def stripe_source(
 
 @dlt.source
 def incremental_stripe_source(
-    endpoints: Tuple[str, ...] = INCREMENTAL_ENDPOINTS,
+    endpoints: Tuple[str, ...],
     stripe_secret_key: str = dlt.secrets.value,
     initial_start_date: Optional[DateTime] = None,
     end_date: Optional[DateTime] = None,

ingestr-0.13.50/ingestr/src/stripe_analytics/settings.py

@@ -0,0 +1,26 @@
+"""Stripe analytics source settings and constants"""
+
+# the most popular endpoints
+# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
+ENDPOINTS = {
+    "subscription": "Subscription",
+    "account": "Account",
+    "coupon": "Coupon",
+    "customer": "Customer",
+    "product": "Product",
+    "price": "Price",
+    "shippingrate": "ShippingRate",
+    "dispute": "Dispute",
+    "subscriptionitem": "SubscriptionItem",
+    "checkoutsession": "CheckoutSession",
+}
+# possible incremental endpoints
+INCREMENTAL_ENDPOINTS = {
+    "event": "Event",
+    "invoice": "Invoice",
+    "balancetransaction": "BalanceTransaction",
+    "charge": "Charge",
+    "applicationfee": "ApplicationFee",
+    "setupattempt": "SetupAttempt",
+    "creditnote": "CreditNote",
+}

ingestr-0.13.50/ingestr/tests/unit/test_smartsheets.py

@@ -0,0 +1,136 @@
+import sys
+import unittest
+from unittest.mock import MagicMock, patch
+
+import smartsheet  # type: ignore
+from smartsheet.models import Cell, Column, Row, Sheet  # type: ignore
+
+from ingestr.src.smartsheets import _get_sheet_data, smartsheet_source
+
+
+def pp(x):
+    print(x, file=sys.stderr)
+
+
+class TestSmartsheetSource(unittest.TestCase):
+    @patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
+    def test_smartsheet_source_success(self, mock_smartsheet_client):
+        # Mock Smartsheet client and its methods
+        mock_client_instance = mock_smartsheet_client.return_value
+
+        # Mock sheet details response
+        mock_sheet_details = Sheet(
+            {
+                "id": 123,
+                "name": "Test Sheet 1",
+                "columns": [
+                    Column(
+                        {"id": 1, "title": "Col A", "type": "TEXT_NUMBER", "index": 0}
+                    ),
+                    Column(
+                        {"id": 2, "title": "Col B", "type": "TEXT_NUMBER", "index": 1}
+                    ),
+                ],
+                "rows": [
+                    Row(
+                        {
+                            "id": 101,
+                            "sheetId": 123,
+                            "cells": [
+                                Cell({"columnId": 1, "value": "r1c1"}),
+                                Cell({"columnId": 2, "value": "r1c2"}),
+                            ],
+                        }
+                    ),
+                    Row(
+                        {
+                            "id": 102,
+                            "sheetId": 123,
+                            "cells": [
+                                Cell({"columnId": 1, "value": "r2c1"}),
+                                Cell({"columnId": 2, "value": "r2c2"}),
+                            ],
+                        }
+                    ),
+                ],
+            }
+        )
+        mock_client_instance.Sheets.get_sheet.return_value = mock_sheet_details
+
+        resource = smartsheet_source(access_token="test_token", sheet_id="123")
+        data = list(resource)
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0], {"Col A": "r1c1", "Col B": "r1c2"})
+        self.assertEqual(data[1], {"Col A": "r2c1", "Col B": "r2c2"})
+
+        mock_smartsheet_client.assert_called_once_with("test_token")
+        mock_client_instance.Sheets.get_sheet.assert_any_call(
+            123, include=["objectValue"]
+        )  # for resource name
+        mock_client_instance.Sheets.get_sheet.assert_any_call(
+            123
+        )  # for _get_sheet_data
+
+    @patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
+    def test_smartsheet_source_api_error(self, mock_smartsheet_client):
+        mock_client_instance = mock_smartsheet_client.return_value
+        mock_client_instance.Sheets.get_sheet.side_effect = (
+            smartsheet.exceptions.ApiError("API Error", 500)
+        )
+
+        with self.assertRaises(smartsheet.exceptions.ApiError):
+            source = smartsheet_source(access_token="test_token", sheet_id="123")
+            # Consume the generator to trigger the API call
+            list(source)
+
+    def test_get_sheet_data(self):
+        mock_smartsheet_client_instance = MagicMock()
+        mock_sheet = Sheet(
+            {
+                "id": 456,
+                "name": "Data Sheet",
+                "columns": [
+                    Column(
+                        {"id": 10, "title": "ID", "type": "TEXT_NUMBER", "index": 0}
+                    ),
+                    Column(
+                        {"id": 20, "title": "Value", "type": "TEXT_NUMBER", "index": 1}
+                    ),
+                ],
+                "rows": [
+                    Row(
+                        {
+                            "id": 201,
+                            "sheetId": 456,
+                            "cells": [
+                                Cell({"columnId": 10, "value": 1}),
+                                Cell({"columnId": 20, "value": "Alpha"}),
+                            ],
+                        }
+                    ),
+                    Row(
+                        {
+                            "id": 202,
+                            "sheetId": 456,
+                            "cells": [
+                                Cell({"columnId": 10, "value": 2}),
+                                Cell({"columnId": 20, "value": "Beta"}),
+                            ],
+                        }
+                    ),
+                ],
+            }
+        )
+        mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
+
+        data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
+        data = list(data_generator)
+
+        self.assertEqual(len(data), 2)
+        self.assertEqual(data[0], {"ID": 1, "Value": "Alpha"})
+        self.assertEqual(data[1], {"ID": 2, "Value": "Beta"})
+        mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
+
+
+if __name__ == "__main__":
+    unittest.main()

{ingestr-0.13.48 → ingestr-0.13.50}/requirements.in

@@ -52,3 +52,4 @@ aiohttp>=3.11.15
 elasticsearch==8.10.1
 sqlalchemy-spanner==1.11.0
 google-cloud-spanner==3.54.0
+smartsheet-python-sdk==3.0.5

{ingestr-0.13.48 → ingestr-0.13.50}/requirements.txt

@@ -55,6 +55,7 @@ certifi==2025.1.31
     #   clickhouse-connect
     #   elastic-transport
     #   requests
+    #   smartsheet-python-sdk
     #   snowflake-connector-python
 cffi==1.17.1
     # via
@@ -402,6 +403,7 @@ python-dateutil==2.9.0.post0
     #   pendulum
     #   pyathena
     #   rudder-sdk-python
+    #   smartsheet-python-sdk
     #   time-machine
 python-dotenv==1.0.1
     # via rudder-sdk-python
@@ -440,6 +442,7 @@ requests==2.32.3
     #   requests-toolbelt
     #   rudder-sdk-python
     #   simple-salesforce
+    #   smartsheet-python-sdk
     #   snowflake-connector-python
     #   stripe
     #   zeep
@@ -450,7 +453,9 @@ requests-oauthlib==1.3.1
     #   asana
     #   google-auth-oauthlib
 requests-toolbelt==1.0.0
-    # via zeep
+    # via
+    #   smartsheet-python-sdk
+    #   zeep
 requirements-parser==0.11.0
     # via dlt
 rich==13.9.4
@@ -488,7 +493,10 @@ six==1.17.0
     #   facebook-business
     #   flatten-json
     #   python-dateutil
+    #   smartsheet-python-sdk
     #   thrift
+smartsheet-python-sdk==3.0.5
+    # via -r requirements.in
 smmap==5.0.2
     # via gitdb
 snowflake-connector-python==3.14.0