ingestr 0.13.48__py3-none-any.whl → 0.13.50__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/main.py +8 -0
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/mongodb/__init__.py +2 -1
- ingestr/src/resource.py +23 -0
- ingestr/src/smartsheets/__init__.py +54 -0
- ingestr/src/solidgate/helpers.py +2 -6
- ingestr/src/sources.py +71 -38
- ingestr/src/stripe_analytics/__init__.py +2 -3
- ingestr/src/stripe_analytics/settings.py +21 -9
- ingestr/tests/unit/test_smartsheets.py +136 -0
- {ingestr-0.13.48.dist-info → ingestr-0.13.50.dist-info}/METADATA +7 -1
- {ingestr-0.13.48.dist-info → ingestr-0.13.50.dist-info}/RECORD +15 -13
- {ingestr-0.13.48.dist-info → ingestr-0.13.50.dist-info}/WHEEL +0 -0
- {ingestr-0.13.48.dist-info → ingestr-0.13.50.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.48.dist-info → ingestr-0.13.50.dist-info}/licenses/LICENSE.md +0 -0
ingestr/main.py
CHANGED
|
@@ -290,6 +290,7 @@ def ingest(
|
|
|
290
290
|
from ingestr.src.destinations import AthenaDestination
|
|
291
291
|
from ingestr.src.factory import SourceDestinationFactory
|
|
292
292
|
from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
|
|
293
|
+
from ingestr.src.sources import MongoDbSource
|
|
293
294
|
|
|
294
295
|
def report_errors(run_info: LoadInfo):
|
|
295
296
|
for load_package in run_info.load_packages:
|
|
@@ -537,6 +538,13 @@ def ingest(
|
|
|
537
538
|
if yield_limit:
|
|
538
539
|
resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
|
|
539
540
|
|
|
541
|
+
if isinstance(source, MongoDbSource):
|
|
542
|
+
from ingestr.src.resource import TypeHintMap
|
|
543
|
+
|
|
544
|
+
resource.for_each(
|
|
545
|
+
dlt_source, lambda x: x.add_map(TypeHintMap().type_hint_map)
|
|
546
|
+
)
|
|
547
|
+
|
|
540
548
|
def col_h(x):
|
|
541
549
|
if column_hints:
|
|
542
550
|
x.apply_hints(columns=column_hints)
|
ingestr/src/buildinfo.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = "v0.13.
|
|
1
|
+
version = "v0.13.50"
|
ingestr/src/mongodb/__init__.py
CHANGED
|
@@ -14,7 +14,7 @@ from .helpers import (
|
|
|
14
14
|
)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
@dlt.source
|
|
17
|
+
@dlt.source(max_table_nesting=0)
|
|
18
18
|
def mongodb(
|
|
19
19
|
connection_url: str = dlt.secrets.value,
|
|
20
20
|
database: Optional[str] = dlt.config.value,
|
|
@@ -75,6 +75,7 @@ def mongodb(
|
|
|
75
75
|
primary_key="_id",
|
|
76
76
|
write_disposition=write_disposition,
|
|
77
77
|
spec=MongoDbCollectionConfiguration,
|
|
78
|
+
max_table_nesting=0,
|
|
78
79
|
)(
|
|
79
80
|
client,
|
|
80
81
|
collection,
|
ingestr/src/resource.py
CHANGED
|
@@ -15,3 +15,26 @@ def for_each(
|
|
|
15
15
|
ex(source.resources[res]) # type: ignore[union-attr]
|
|
16
16
|
else:
|
|
17
17
|
ex(source) # type: ignore[arg-type]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TypeHintMap:
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.handled_typehints = False
|
|
23
|
+
|
|
24
|
+
def type_hint_map(self, item):
|
|
25
|
+
if self.handled_typehints:
|
|
26
|
+
return item
|
|
27
|
+
|
|
28
|
+
array_cols = []
|
|
29
|
+
for col in item:
|
|
30
|
+
if isinstance(item[col], (list, tuple)):
|
|
31
|
+
array_cols.append(col)
|
|
32
|
+
if array_cols:
|
|
33
|
+
import dlt
|
|
34
|
+
|
|
35
|
+
source = dlt.current.source()
|
|
36
|
+
columns = [{"name": col, "data_type": "json"} for col in array_cols]
|
|
37
|
+
for_each(source, lambda x: x.apply_hints(columns=columns))
|
|
38
|
+
|
|
39
|
+
self.handled_typehints = True
|
|
40
|
+
return item
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from typing import Iterable
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
import smartsheet # type: ignore
|
|
5
|
+
from dlt.extract import DltResource
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dlt.source
|
|
9
|
+
def smartsheet_source(
|
|
10
|
+
access_token: str,
|
|
11
|
+
sheet_id: str,
|
|
12
|
+
) -> Iterable[DltResource]:
|
|
13
|
+
"""
|
|
14
|
+
A DLT source for Smartsheet.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
access_token: The Smartsheet API access token.
|
|
18
|
+
sheet_id: The ID of the sheet to load.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
An iterable of DLT resources.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
# Initialize Smartsheet client
|
|
25
|
+
smartsheet_client = smartsheet.Smartsheet(access_token)
|
|
26
|
+
smartsheet_client.errors_as_exceptions(True)
|
|
27
|
+
|
|
28
|
+
# The SDK expects sheet_id to be an int
|
|
29
|
+
sheet_id_int = int(sheet_id)
|
|
30
|
+
# Sanitize the sheet name to be a valid resource name
|
|
31
|
+
# We get objectValue to ensure `name` attribute is populated for the sheet
|
|
32
|
+
sheet_details = smartsheet_client.Sheets.get_sheet(
|
|
33
|
+
sheet_id_int, include=["objectValue"]
|
|
34
|
+
)
|
|
35
|
+
sheet_name = sheet_details.name
|
|
36
|
+
resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
|
|
37
|
+
|
|
38
|
+
yield dlt.resource(
|
|
39
|
+
_get_sheet_data(smartsheet_client, sheet_id_int),
|
|
40
|
+
name=resource_name,
|
|
41
|
+
write_disposition="replace",
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
|
|
46
|
+
"""Helper function to get all rows from a sheet."""
|
|
47
|
+
sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
|
|
48
|
+
# Transform rows to a list of dictionaries
|
|
49
|
+
column_titles = [col.title for col in sheet.columns]
|
|
50
|
+
for row in sheet.rows:
|
|
51
|
+
row_data = {}
|
|
52
|
+
for i, cell in enumerate(row.cells):
|
|
53
|
+
row_data[column_titles[i]] = cell.value
|
|
54
|
+
yield row_data
|
ingestr/src/solidgate/helpers.py
CHANGED
|
@@ -50,18 +50,14 @@ class SolidgateClient:
|
|
|
50
50
|
data = response_json["subscriptions"]
|
|
51
51
|
for _, value in data.items():
|
|
52
52
|
if "updated_at" in value:
|
|
53
|
-
value["updated_at"] = pendulum.parse(
|
|
54
|
-
value["updated_at"]
|
|
55
|
-
)
|
|
53
|
+
value["updated_at"] = pendulum.parse(value["updated_at"])
|
|
56
54
|
yield value
|
|
57
55
|
|
|
58
56
|
else:
|
|
59
57
|
data = response_json["orders"]
|
|
60
58
|
for value in data:
|
|
61
59
|
if "updated_at" in value:
|
|
62
|
-
value["updated_at"] = pendulum.parse(
|
|
63
|
-
value["updated_at"]
|
|
64
|
-
)
|
|
60
|
+
value["updated_at"] = pendulum.parse(value["updated_at"])
|
|
65
61
|
yield value
|
|
66
62
|
|
|
67
63
|
next_page_iterator = response_json.get("metadata", {}).get(
|
ingestr/src/sources.py
CHANGED
|
@@ -388,6 +388,7 @@ class MongoDbSource:
|
|
|
388
388
|
parallel=True,
|
|
389
389
|
incremental=incremental,
|
|
390
390
|
)
|
|
391
|
+
table_instance.max_table_nesting = 1
|
|
391
392
|
|
|
392
393
|
return table_instance
|
|
393
394
|
|
|
@@ -697,46 +698,47 @@ class StripeAnalyticsSource:
|
|
|
697
698
|
if not api_key:
|
|
698
699
|
raise ValueError("api_key in the URI is required to connect to Stripe")
|
|
699
700
|
|
|
700
|
-
|
|
701
|
-
if table == "balancetransaction":
|
|
702
|
-
table = "BalanceTransaction"
|
|
703
|
-
else:
|
|
704
|
-
table = table.capitalize()
|
|
705
|
-
|
|
706
|
-
if table in [
|
|
707
|
-
"Subscription",
|
|
708
|
-
"Account",
|
|
709
|
-
"Coupon",
|
|
710
|
-
"Customer",
|
|
711
|
-
"Product",
|
|
712
|
-
"Price",
|
|
713
|
-
"BalanceTransaction",
|
|
714
|
-
"Invoice",
|
|
715
|
-
"Event",
|
|
716
|
-
"Charge",
|
|
717
|
-
]:
|
|
718
|
-
endpoint = table
|
|
719
|
-
else:
|
|
720
|
-
raise ValueError(
|
|
721
|
-
f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
722
|
-
)
|
|
723
|
-
|
|
724
|
-
date_args = {}
|
|
725
|
-
if kwargs.get("interval_start"):
|
|
726
|
-
date_args["start_date"] = kwargs.get("interval_start")
|
|
727
|
-
|
|
728
|
-
if kwargs.get("interval_end"):
|
|
729
|
-
date_args["end_date"] = kwargs.get("interval_end")
|
|
701
|
+
table = table.lower()
|
|
730
702
|
|
|
731
|
-
from ingestr.src.stripe_analytics import
|
|
703
|
+
from ingestr.src.stripe_analytics.settings import (
|
|
704
|
+
ENDPOINTS,
|
|
705
|
+
INCREMENTAL_ENDPOINTS,
|
|
706
|
+
)
|
|
732
707
|
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
708
|
+
if table in ENDPOINTS:
|
|
709
|
+
endpoint = ENDPOINTS[table]
|
|
710
|
+
from ingestr.src.stripe_analytics import stripe_source
|
|
711
|
+
|
|
712
|
+
return stripe_source(
|
|
713
|
+
endpoints=[
|
|
714
|
+
endpoint,
|
|
715
|
+
],
|
|
716
|
+
stripe_secret_key=api_key[0],
|
|
717
|
+
start_date=kwargs.get("interval_start", None),
|
|
718
|
+
end_date=kwargs.get("interval_end", None),
|
|
719
|
+
).with_resources(endpoint)
|
|
720
|
+
|
|
721
|
+
elif table in INCREMENTAL_ENDPOINTS:
|
|
722
|
+
endpoint = INCREMENTAL_ENDPOINTS[table]
|
|
723
|
+
from ingestr.src.stripe_analytics import incremental_stripe_source
|
|
724
|
+
|
|
725
|
+
def nullable_date(date_str: Optional[str]):
|
|
726
|
+
if date_str:
|
|
727
|
+
return ensure_pendulum_datetime(date_str)
|
|
728
|
+
return None
|
|
729
|
+
|
|
730
|
+
return incremental_stripe_source(
|
|
731
|
+
endpoints=[
|
|
732
|
+
endpoint,
|
|
733
|
+
],
|
|
734
|
+
stripe_secret_key=api_key[0],
|
|
735
|
+
initial_start_date=nullable_date(kwargs.get("interval_start", None)),
|
|
736
|
+
end_date=nullable_date(kwargs.get("interval_end", None)),
|
|
737
|
+
).with_resources(endpoint)
|
|
738
|
+
|
|
739
|
+
raise ValueError(
|
|
740
|
+
f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
741
|
+
)
|
|
740
742
|
|
|
741
743
|
|
|
742
744
|
class FacebookAdsSource:
|
|
@@ -2433,6 +2435,37 @@ class AttioSource:
|
|
|
2433
2435
|
raise UnsupportedResourceError(table_name, "Attio")
|
|
2434
2436
|
|
|
2435
2437
|
|
|
2438
|
+
class SmartsheetSource:
|
|
2439
|
+
def handles_incrementality(self) -> bool:
|
|
2440
|
+
return False
|
|
2441
|
+
|
|
2442
|
+
# smartsheet://?access_token=<access_token>
|
|
2443
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2444
|
+
if kwargs.get("incremental_key"):
|
|
2445
|
+
raise ValueError("Incremental loads are not supported for Smartsheet")
|
|
2446
|
+
|
|
2447
|
+
if not table:
|
|
2448
|
+
raise ValueError(
|
|
2449
|
+
"Source table (sheet_id) is required to connect to Smartsheet"
|
|
2450
|
+
)
|
|
2451
|
+
|
|
2452
|
+
source_parts = urlparse(uri)
|
|
2453
|
+
source_fields = parse_qs(source_parts.query)
|
|
2454
|
+
access_token = source_fields.get("access_token")
|
|
2455
|
+
|
|
2456
|
+
if not access_token:
|
|
2457
|
+
raise ValueError(
|
|
2458
|
+
"access_token in the URI is required to connect to Smartsheet"
|
|
2459
|
+
)
|
|
2460
|
+
|
|
2461
|
+
from ingestr.src.smartsheets import smartsheet_source
|
|
2462
|
+
|
|
2463
|
+
return smartsheet_source(
|
|
2464
|
+
access_token=access_token[0],
|
|
2465
|
+
sheet_id=table, # table is now a single sheet_id
|
|
2466
|
+
)
|
|
2467
|
+
|
|
2468
|
+
|
|
2436
2469
|
class SolidgateSource:
|
|
2437
2470
|
def handles_incrementality(self) -> bool:
|
|
2438
2471
|
return True
|
|
@@ -8,12 +8,11 @@ from dlt.sources import DltResource
|
|
|
8
8
|
from pendulum import DateTime
|
|
9
9
|
|
|
10
10
|
from .helpers import pagination, transform_date
|
|
11
|
-
from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
@dlt.source(max_table_nesting=0)
|
|
15
14
|
def stripe_source(
|
|
16
|
-
endpoints: Tuple[str, ...]
|
|
15
|
+
endpoints: Tuple[str, ...],
|
|
17
16
|
stripe_secret_key: str = dlt.secrets.value,
|
|
18
17
|
start_date: Optional[DateTime] = None,
|
|
19
18
|
end_date: Optional[DateTime] = None,
|
|
@@ -53,7 +52,7 @@ def stripe_source(
|
|
|
53
52
|
|
|
54
53
|
@dlt.source
|
|
55
54
|
def incremental_stripe_source(
|
|
56
|
-
endpoints: Tuple[str, ...]
|
|
55
|
+
endpoints: Tuple[str, ...],
|
|
57
56
|
stripe_secret_key: str = dlt.secrets.value,
|
|
58
57
|
initial_start_date: Optional[DateTime] = None,
|
|
59
58
|
end_date: Optional[DateTime] = None,
|
|
@@ -2,13 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
# the most popular endpoints
|
|
4
4
|
# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
|
|
5
|
-
ENDPOINTS =
|
|
6
|
-
"Subscription",
|
|
7
|
-
"Account",
|
|
8
|
-
"Coupon",
|
|
9
|
-
"Customer",
|
|
10
|
-
"Product",
|
|
11
|
-
"Price",
|
|
12
|
-
|
|
5
|
+
ENDPOINTS = {
|
|
6
|
+
"subscription": "Subscription",
|
|
7
|
+
"account": "Account",
|
|
8
|
+
"coupon": "Coupon",
|
|
9
|
+
"customer": "Customer",
|
|
10
|
+
"product": "Product",
|
|
11
|
+
"price": "Price",
|
|
12
|
+
"shippingrate": "ShippingRate",
|
|
13
|
+
"dispute": "Dispute",
|
|
14
|
+
"subscriptionitem": "SubscriptionItem",
|
|
15
|
+
"checkoutsession": "CheckoutSession",
|
|
16
|
+
}
|
|
13
17
|
# possible incremental endpoints
|
|
14
|
-
INCREMENTAL_ENDPOINTS =
|
|
18
|
+
INCREMENTAL_ENDPOINTS = {
|
|
19
|
+
"event": "Event",
|
|
20
|
+
"invoice": "Invoice",
|
|
21
|
+
"balancetransaction": "BalanceTransaction",
|
|
22
|
+
"charge": "Charge",
|
|
23
|
+
"applicationfee": "ApplicationFee",
|
|
24
|
+
"setupattempt": "SetupAttempt",
|
|
25
|
+
"creditnote": "CreditNote",
|
|
26
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import unittest
|
|
3
|
+
from unittest.mock import MagicMock, patch
|
|
4
|
+
|
|
5
|
+
import smartsheet # type: ignore
|
|
6
|
+
from smartsheet.models import Cell, Column, Row, Sheet # type: ignore
|
|
7
|
+
|
|
8
|
+
from ingestr.src.smartsheets import _get_sheet_data, smartsheet_source
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def pp(x):
|
|
12
|
+
print(x, file=sys.stderr)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TestSmartsheetSource(unittest.TestCase):
|
|
16
|
+
@patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
|
|
17
|
+
def test_smartsheet_source_success(self, mock_smartsheet_client):
|
|
18
|
+
# Mock Smartsheet client and its methods
|
|
19
|
+
mock_client_instance = mock_smartsheet_client.return_value
|
|
20
|
+
|
|
21
|
+
# Mock sheet details response
|
|
22
|
+
mock_sheet_details = Sheet(
|
|
23
|
+
{
|
|
24
|
+
"id": 123,
|
|
25
|
+
"name": "Test Sheet 1",
|
|
26
|
+
"columns": [
|
|
27
|
+
Column(
|
|
28
|
+
{"id": 1, "title": "Col A", "type": "TEXT_NUMBER", "index": 0}
|
|
29
|
+
),
|
|
30
|
+
Column(
|
|
31
|
+
{"id": 2, "title": "Col B", "type": "TEXT_NUMBER", "index": 1}
|
|
32
|
+
),
|
|
33
|
+
],
|
|
34
|
+
"rows": [
|
|
35
|
+
Row(
|
|
36
|
+
{
|
|
37
|
+
"id": 101,
|
|
38
|
+
"sheetId": 123,
|
|
39
|
+
"cells": [
|
|
40
|
+
Cell({"columnId": 1, "value": "r1c1"}),
|
|
41
|
+
Cell({"columnId": 2, "value": "r1c2"}),
|
|
42
|
+
],
|
|
43
|
+
}
|
|
44
|
+
),
|
|
45
|
+
Row(
|
|
46
|
+
{
|
|
47
|
+
"id": 102,
|
|
48
|
+
"sheetId": 123,
|
|
49
|
+
"cells": [
|
|
50
|
+
Cell({"columnId": 1, "value": "r2c1"}),
|
|
51
|
+
Cell({"columnId": 2, "value": "r2c2"}),
|
|
52
|
+
],
|
|
53
|
+
}
|
|
54
|
+
),
|
|
55
|
+
],
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
mock_client_instance.Sheets.get_sheet.return_value = mock_sheet_details
|
|
59
|
+
|
|
60
|
+
resource = smartsheet_source(access_token="test_token", sheet_id="123")
|
|
61
|
+
data = list(resource)
|
|
62
|
+
self.assertEqual(len(data), 2)
|
|
63
|
+
self.assertEqual(data[0], {"Col A": "r1c1", "Col B": "r1c2"})
|
|
64
|
+
self.assertEqual(data[1], {"Col A": "r2c1", "Col B": "r2c2"})
|
|
65
|
+
|
|
66
|
+
mock_smartsheet_client.assert_called_once_with("test_token")
|
|
67
|
+
mock_client_instance.Sheets.get_sheet.assert_any_call(
|
|
68
|
+
123, include=["objectValue"]
|
|
69
|
+
) # for resource name
|
|
70
|
+
mock_client_instance.Sheets.get_sheet.assert_any_call(
|
|
71
|
+
123
|
|
72
|
+
) # for _get_sheet_data
|
|
73
|
+
|
|
74
|
+
@patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
|
|
75
|
+
def test_smartsheet_source_api_error(self, mock_smartsheet_client):
|
|
76
|
+
mock_client_instance = mock_smartsheet_client.return_value
|
|
77
|
+
mock_client_instance.Sheets.get_sheet.side_effect = (
|
|
78
|
+
smartsheet.exceptions.ApiError("API Error", 500)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
with self.assertRaises(smartsheet.exceptions.ApiError):
|
|
82
|
+
source = smartsheet_source(access_token="test_token", sheet_id="123")
|
|
83
|
+
# Consume the generator to trigger the API call
|
|
84
|
+
list(source)
|
|
85
|
+
|
|
86
|
+
def test_get_sheet_data(self):
|
|
87
|
+
mock_smartsheet_client_instance = MagicMock()
|
|
88
|
+
mock_sheet = Sheet(
|
|
89
|
+
{
|
|
90
|
+
"id": 456,
|
|
91
|
+
"name": "Data Sheet",
|
|
92
|
+
"columns": [
|
|
93
|
+
Column(
|
|
94
|
+
{"id": 10, "title": "ID", "type": "TEXT_NUMBER", "index": 0}
|
|
95
|
+
),
|
|
96
|
+
Column(
|
|
97
|
+
{"id": 20, "title": "Value", "type": "TEXT_NUMBER", "index": 1}
|
|
98
|
+
),
|
|
99
|
+
],
|
|
100
|
+
"rows": [
|
|
101
|
+
Row(
|
|
102
|
+
{
|
|
103
|
+
"id": 201,
|
|
104
|
+
"sheetId": 456,
|
|
105
|
+
"cells": [
|
|
106
|
+
Cell({"columnId": 10, "value": 1}),
|
|
107
|
+
Cell({"columnId": 20, "value": "Alpha"}),
|
|
108
|
+
],
|
|
109
|
+
}
|
|
110
|
+
),
|
|
111
|
+
Row(
|
|
112
|
+
{
|
|
113
|
+
"id": 202,
|
|
114
|
+
"sheetId": 456,
|
|
115
|
+
"cells": [
|
|
116
|
+
Cell({"columnId": 10, "value": 2}),
|
|
117
|
+
Cell({"columnId": 20, "value": "Beta"}),
|
|
118
|
+
],
|
|
119
|
+
}
|
|
120
|
+
),
|
|
121
|
+
],
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
|
|
125
|
+
|
|
126
|
+
data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
|
|
127
|
+
data = list(data_generator)
|
|
128
|
+
|
|
129
|
+
self.assertEqual(len(data), 2)
|
|
130
|
+
self.assertEqual(data[0], {"ID": 1, "Value": "Alpha"})
|
|
131
|
+
self.assertEqual(data[1], {"ID": 2, "Value": "Beta"})
|
|
132
|
+
mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
unittest.main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.50
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -163,6 +163,7 @@ Requires-Dist: shellingham==1.5.4
|
|
|
163
163
|
Requires-Dist: simple-salesforce==1.12.6
|
|
164
164
|
Requires-Dist: simplejson==3.20.1
|
|
165
165
|
Requires-Dist: six==1.17.0
|
|
166
|
+
Requires-Dist: smartsheet-python-sdk==3.0.5
|
|
166
167
|
Requires-Dist: smmap==5.0.2
|
|
167
168
|
Requires-Dist: snowflake-connector-python==3.14.0
|
|
168
169
|
Requires-Dist: snowflake-sqlalchemy==1.6.1
|
|
@@ -467,6 +468,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
467
468
|
<td>✅</td>
|
|
468
469
|
<td>-</td>
|
|
469
470
|
</tr>
|
|
471
|
+
<tr>
|
|
472
|
+
<td>Smartsheet</td>
|
|
473
|
+
<td>✅</td>
|
|
474
|
+
<td>-</td>
|
|
475
|
+
</tr>
|
|
470
476
|
<tr>
|
|
471
477
|
<td>Stripe</td>
|
|
472
478
|
<td>✅</td>
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
|
|
2
|
-
ingestr/main.py,sha256=
|
|
2
|
+
ingestr/main.py,sha256=rHxHQAbd0ccW2e2kQSWlv7-5qcc2ZB6Eh3vyjm4Nzns,25550
|
|
3
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
4
4
|
ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
|
|
5
|
-
ingestr/src/buildinfo.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=SIygmTZiIS9bHiKi-Fr3gXHmpPssVhCeEF_SCAtF8Ww,21
|
|
6
6
|
ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
|
|
7
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
8
8
|
ingestr/src/factory.py,sha256=FXWJLFfBsKMzUwtsyaaruZU-_OLFKivobj6Olse9vSI,5741
|
|
@@ -10,8 +10,8 @@ ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
|
|
|
10
10
|
ingestr/src/http_client.py,sha256=UwPiv95EfHPdT4525xeLFJ1AYlf-cyaHKRU-2QnZt2o,435
|
|
11
11
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
12
12
|
ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
|
|
13
|
-
ingestr/src/resource.py,sha256=
|
|
14
|
-
ingestr/src/sources.py,sha256=
|
|
13
|
+
ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
|
|
14
|
+
ingestr/src/sources.py,sha256=SnFxil6gU5RIYiSUfE6Upq2se0YeRSuTIKK3Jxn-YKI,86587
|
|
15
15
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
16
16
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
17
17
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
@@ -82,7 +82,7 @@ ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYH
|
|
|
82
82
|
ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
|
|
83
83
|
ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
|
|
84
84
|
ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
|
|
85
|
-
ingestr/src/mongodb/__init__.py,sha256=
|
|
85
|
+
ingestr/src/mongodb/__init__.py,sha256=T-RYPS_skl_2gNVfYWWXan2bVQYmm0bFBcCCqG5ejvg,7275
|
|
86
86
|
ingestr/src/mongodb/helpers.py,sha256=H0GpOK3bPBhFWBEhJZOjywUBdzih6MOpmyVO_cKSN14,24178
|
|
87
87
|
ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
|
|
88
88
|
ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
|
|
@@ -108,13 +108,14 @@ ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSs
|
|
|
108
108
|
ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
|
|
109
109
|
ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
|
|
110
110
|
ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
|
|
111
|
+
ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
|
|
111
112
|
ingestr/src/solidgate/__init__.py,sha256=vpoXu0Ox3zE_WPSzdsA6iUG1_XBa9OaA5F7eFBbZYuQ,2819
|
|
112
|
-
ingestr/src/solidgate/helpers.py,sha256=
|
|
113
|
+
ingestr/src/solidgate/helpers.py,sha256=_PuHmKZ-jpNEsPxRgXCzu39PsaygVDCpnoMTZAYSpHE,2432
|
|
113
114
|
ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
114
115
|
ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
|
|
115
|
-
ingestr/src/stripe_analytics/__init__.py,sha256=
|
|
116
|
+
ingestr/src/stripe_analytics/__init__.py,sha256=FBkZu5op5Z-FceEi4zG7qcAgZfUYJRPMVPPrPMjvmXw,4502
|
|
116
117
|
ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
|
|
117
|
-
ingestr/src/stripe_analytics/settings.py,sha256=
|
|
118
|
+
ingestr/src/stripe_analytics/settings.py,sha256=fA2j_6FquEmyRyB799P4SncwLwK1S1u9WFNjbzu91kY,786
|
|
118
119
|
ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
|
|
119
120
|
ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
|
|
120
121
|
ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
|
|
@@ -133,8 +134,9 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
133
134
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
134
135
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
135
136
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
136
|
-
ingestr
|
|
137
|
-
ingestr-0.13.
|
|
138
|
-
ingestr-0.13.
|
|
139
|
-
ingestr-0.13.
|
|
140
|
-
ingestr-0.13.
|
|
137
|
+
ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
|
|
138
|
+
ingestr-0.13.50.dist-info/METADATA,sha256=cYNN5jAD-1Fistktk9x6ggyzbHIbmKEoOkwUgkLLosI,13983
|
|
139
|
+
ingestr-0.13.50.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
140
|
+
ingestr-0.13.50.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
141
|
+
ingestr-0.13.50.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
142
|
+
ingestr-0.13.50.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|