ingestr 0.13.47__py3-none-any.whl → 0.13.49__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/main.py +8 -0
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/factory.py +2 -0
- ingestr/src/mongodb/__init__.py +2 -1
- ingestr/src/resource.py +23 -0
- ingestr/src/smartsheets/__init__.py +54 -0
- ingestr/src/solidgate/__init__.py +97 -0
- ingestr/src/solidgate/helpers.py +74 -0
- ingestr/src/sources.py +113 -38
- ingestr/src/stripe_analytics/__init__.py +2 -3
- ingestr/src/stripe_analytics/settings.py +21 -9
- ingestr/tests/unit/test_smartsheets.py +136 -0
- {ingestr-0.13.47.dist-info → ingestr-0.13.49.dist-info}/METADATA +7 -1
- {ingestr-0.13.47.dist-info → ingestr-0.13.49.dist-info}/RECORD +17 -13
- {ingestr-0.13.47.dist-info → ingestr-0.13.49.dist-info}/WHEEL +0 -0
- {ingestr-0.13.47.dist-info → ingestr-0.13.49.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.47.dist-info → ingestr-0.13.49.dist-info}/licenses/LICENSE.md +0 -0
ingestr/main.py
CHANGED
|
@@ -290,6 +290,7 @@ def ingest(
|
|
|
290
290
|
from ingestr.src.destinations import AthenaDestination
|
|
291
291
|
from ingestr.src.factory import SourceDestinationFactory
|
|
292
292
|
from ingestr.src.filters import cast_set_to_list, handle_mysql_empty_dates
|
|
293
|
+
from ingestr.src.sources import MongoDbSource
|
|
293
294
|
|
|
294
295
|
def report_errors(run_info: LoadInfo):
|
|
295
296
|
for load_package in run_info.load_packages:
|
|
@@ -537,6 +538,13 @@ def ingest(
|
|
|
537
538
|
if yield_limit:
|
|
538
539
|
resource.for_each(dlt_source, lambda x: x.add_limit(yield_limit))
|
|
539
540
|
|
|
541
|
+
if isinstance(source, MongoDbSource):
|
|
542
|
+
from ingestr.src.resource import TypeHintMap
|
|
543
|
+
|
|
544
|
+
resource.for_each(
|
|
545
|
+
dlt_source, lambda x: x.add_map(TypeHintMap().type_hint_map)
|
|
546
|
+
)
|
|
547
|
+
|
|
540
548
|
def col_h(x):
|
|
541
549
|
if column_hints:
|
|
542
550
|
x.apply_hints(columns=column_hints)
|
ingestr/src/buildinfo.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = "v0.13.
|
|
1
|
+
version = "v0.13.49"
|
ingestr/src/factory.py
CHANGED
|
@@ -54,6 +54,7 @@ from ingestr.src.sources import (
|
|
|
54
54
|
SalesforceSource,
|
|
55
55
|
ShopifySource,
|
|
56
56
|
SlackSource,
|
|
57
|
+
SolidgateSource,
|
|
57
58
|
SqlSource,
|
|
58
59
|
StripeAnalyticsSource,
|
|
59
60
|
TikTokSource,
|
|
@@ -159,6 +160,7 @@ class SourceDestinationFactory:
|
|
|
159
160
|
"phantombuster": PhantombusterSource,
|
|
160
161
|
"elasticsearch": ElasticsearchSource,
|
|
161
162
|
"attio": AttioSource,
|
|
163
|
+
"solidgate": SolidgateSource,
|
|
162
164
|
}
|
|
163
165
|
destinations: Dict[str, Type[DestinationProtocol]] = {
|
|
164
166
|
"bigquery": BigQueryDestination,
|
ingestr/src/mongodb/__init__.py
CHANGED
|
@@ -14,7 +14,7 @@ from .helpers import (
|
|
|
14
14
|
)
|
|
15
15
|
|
|
16
16
|
|
|
17
|
-
@dlt.source
|
|
17
|
+
@dlt.source(max_table_nesting=0)
|
|
18
18
|
def mongodb(
|
|
19
19
|
connection_url: str = dlt.secrets.value,
|
|
20
20
|
database: Optional[str] = dlt.config.value,
|
|
@@ -75,6 +75,7 @@ def mongodb(
|
|
|
75
75
|
primary_key="_id",
|
|
76
76
|
write_disposition=write_disposition,
|
|
77
77
|
spec=MongoDbCollectionConfiguration,
|
|
78
|
+
max_table_nesting=0,
|
|
78
79
|
)(
|
|
79
80
|
client,
|
|
80
81
|
collection,
|
ingestr/src/resource.py
CHANGED
|
@@ -15,3 +15,26 @@ def for_each(
|
|
|
15
15
|
ex(source.resources[res]) # type: ignore[union-attr]
|
|
16
16
|
else:
|
|
17
17
|
ex(source) # type: ignore[arg-type]
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class TypeHintMap:
|
|
21
|
+
def __init__(self):
|
|
22
|
+
self.handled_typehints = False
|
|
23
|
+
|
|
24
|
+
def type_hint_map(self, item):
|
|
25
|
+
if self.handled_typehints:
|
|
26
|
+
return item
|
|
27
|
+
|
|
28
|
+
array_cols = []
|
|
29
|
+
for col in item:
|
|
30
|
+
if isinstance(item[col], (list, tuple)):
|
|
31
|
+
array_cols.append(col)
|
|
32
|
+
if array_cols:
|
|
33
|
+
import dlt
|
|
34
|
+
|
|
35
|
+
source = dlt.current.source()
|
|
36
|
+
columns = [{"name": col, "data_type": "json"} for col in array_cols]
|
|
37
|
+
for_each(source, lambda x: x.apply_hints(columns=columns))
|
|
38
|
+
|
|
39
|
+
self.handled_typehints = True
|
|
40
|
+
return item
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
from typing import Iterable
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
import smartsheet # type: ignore
|
|
5
|
+
from dlt.extract import DltResource
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dlt.source
|
|
9
|
+
def smartsheet_source(
|
|
10
|
+
access_token: str,
|
|
11
|
+
sheet_id: str,
|
|
12
|
+
) -> Iterable[DltResource]:
|
|
13
|
+
"""
|
|
14
|
+
A DLT source for Smartsheet.
|
|
15
|
+
|
|
16
|
+
Args:
|
|
17
|
+
access_token: The Smartsheet API access token.
|
|
18
|
+
sheet_id: The ID of the sheet to load.
|
|
19
|
+
|
|
20
|
+
Returns:
|
|
21
|
+
An iterable of DLT resources.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
# Initialize Smartsheet client
|
|
25
|
+
smartsheet_client = smartsheet.Smartsheet(access_token)
|
|
26
|
+
smartsheet_client.errors_as_exceptions(True)
|
|
27
|
+
|
|
28
|
+
# The SDK expects sheet_id to be an int
|
|
29
|
+
sheet_id_int = int(sheet_id)
|
|
30
|
+
# Sanitize the sheet name to be a valid resource name
|
|
31
|
+
# We get objectValue to ensure `name` attribute is populated for the sheet
|
|
32
|
+
sheet_details = smartsheet_client.Sheets.get_sheet(
|
|
33
|
+
sheet_id_int, include=["objectValue"]
|
|
34
|
+
)
|
|
35
|
+
sheet_name = sheet_details.name
|
|
36
|
+
resource_name = f"sheet_{sheet_name.replace(' ', '_').lower()}"
|
|
37
|
+
|
|
38
|
+
yield dlt.resource(
|
|
39
|
+
_get_sheet_data(smartsheet_client, sheet_id_int),
|
|
40
|
+
name=resource_name,
|
|
41
|
+
write_disposition="replace",
|
|
42
|
+
)
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def _get_sheet_data(smartsheet_client: smartsheet.Smartsheet, sheet_id: int):
|
|
46
|
+
"""Helper function to get all rows from a sheet."""
|
|
47
|
+
sheet = smartsheet_client.Sheets.get_sheet(sheet_id)
|
|
48
|
+
# Transform rows to a list of dictionaries
|
|
49
|
+
column_titles = [col.title for col in sheet.columns]
|
|
50
|
+
for row in sheet.rows:
|
|
51
|
+
row_data = {}
|
|
52
|
+
for i, cell in enumerate(row.cells):
|
|
53
|
+
row_data[column_titles[i]] = cell.value
|
|
54
|
+
yield row_data
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
from typing import Iterable, Iterator
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
import pendulum
|
|
5
|
+
from dlt.sources import DltResource
|
|
6
|
+
|
|
7
|
+
from .helpers import SolidgateClient
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dlt.source(max_table_nesting=0)
|
|
11
|
+
def solidgate_source(
|
|
12
|
+
start_date: pendulum.DateTime,
|
|
13
|
+
end_date: pendulum.DateTime | None,
|
|
14
|
+
public_key: str,
|
|
15
|
+
secret_key: str,
|
|
16
|
+
) -> Iterable[DltResource]:
|
|
17
|
+
solidgate_client = SolidgateClient(public_key, secret_key)
|
|
18
|
+
|
|
19
|
+
@dlt.resource(
|
|
20
|
+
name="subscriptions",
|
|
21
|
+
write_disposition="merge",
|
|
22
|
+
primary_key="id",
|
|
23
|
+
columns={
|
|
24
|
+
"created_at": {"data_type": "timestamp", "partition": True},
|
|
25
|
+
},
|
|
26
|
+
)
|
|
27
|
+
def fetch_all_subscriptions(
|
|
28
|
+
dateTime=dlt.sources.incremental(
|
|
29
|
+
"updated_at",
|
|
30
|
+
initial_value=start_date,
|
|
31
|
+
end_value=end_date,
|
|
32
|
+
range_start="closed",
|
|
33
|
+
range_end="closed",
|
|
34
|
+
),
|
|
35
|
+
) -> Iterator[dict]:
|
|
36
|
+
path = "subscriptions"
|
|
37
|
+
if dateTime.end_value is None:
|
|
38
|
+
end_dt = pendulum.now(tz="UTC")
|
|
39
|
+
else:
|
|
40
|
+
end_dt = dateTime.end_value
|
|
41
|
+
|
|
42
|
+
start_dt = dateTime.last_value
|
|
43
|
+
yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
|
|
44
|
+
|
|
45
|
+
@dlt.resource(
|
|
46
|
+
name="apm-orders",
|
|
47
|
+
write_disposition="merge",
|
|
48
|
+
primary_key="order_id",
|
|
49
|
+
columns={
|
|
50
|
+
"created_at": {"data_type": "timestamp", "partition": True},
|
|
51
|
+
},
|
|
52
|
+
)
|
|
53
|
+
def fetch_apm_orders(
|
|
54
|
+
dateTime=dlt.sources.incremental(
|
|
55
|
+
"updated_at",
|
|
56
|
+
initial_value=start_date,
|
|
57
|
+
end_value=end_date,
|
|
58
|
+
range_start="closed",
|
|
59
|
+
range_end="closed",
|
|
60
|
+
),
|
|
61
|
+
) -> Iterator[dict]:
|
|
62
|
+
path = "apm-orders"
|
|
63
|
+
if dateTime.end_value is None:
|
|
64
|
+
end_dt = pendulum.now(tz="UTC")
|
|
65
|
+
else:
|
|
66
|
+
end_dt = dateTime.end_value
|
|
67
|
+
|
|
68
|
+
start_dt = dateTime.last_value
|
|
69
|
+
yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
|
|
70
|
+
|
|
71
|
+
@dlt.resource(
|
|
72
|
+
name="card-orders",
|
|
73
|
+
write_disposition="merge",
|
|
74
|
+
primary_key="order_id",
|
|
75
|
+
columns={
|
|
76
|
+
"created_at": {"data_type": "timestamp", "partition": True},
|
|
77
|
+
},
|
|
78
|
+
)
|
|
79
|
+
def fetch_card_orders(
|
|
80
|
+
dateTime=dlt.sources.incremental(
|
|
81
|
+
"updated_at",
|
|
82
|
+
initial_value=start_date,
|
|
83
|
+
end_value=end_date,
|
|
84
|
+
range_start="closed",
|
|
85
|
+
range_end="closed",
|
|
86
|
+
),
|
|
87
|
+
) -> Iterator[dict]:
|
|
88
|
+
path = "card-orders"
|
|
89
|
+
if dateTime.end_value is None:
|
|
90
|
+
end_dt = pendulum.now(tz="UTC")
|
|
91
|
+
else:
|
|
92
|
+
end_dt = dateTime.end_value
|
|
93
|
+
|
|
94
|
+
start_dt = dateTime.last_value
|
|
95
|
+
yield solidgate_client.fetch_data(path, date_from=start_dt, date_to=end_dt)
|
|
96
|
+
|
|
97
|
+
return fetch_all_subscriptions, fetch_apm_orders, fetch_card_orders
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import base64
|
|
2
|
+
import hashlib
|
|
3
|
+
import hmac
|
|
4
|
+
import json
|
|
5
|
+
|
|
6
|
+
import pendulum
|
|
7
|
+
|
|
8
|
+
from ingestr.src.http_client import create_client
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class SolidgateClient:
|
|
12
|
+
def __init__(self, public_key, secret_key):
|
|
13
|
+
self.base_url = "https://reports.solidgate.com/api/v1"
|
|
14
|
+
self.public_key = public_key
|
|
15
|
+
self.secret_key = secret_key
|
|
16
|
+
self.client = create_client()
|
|
17
|
+
|
|
18
|
+
def fetch_data(
|
|
19
|
+
self,
|
|
20
|
+
path: str,
|
|
21
|
+
date_from: pendulum.DateTime,
|
|
22
|
+
date_to: pendulum.DateTime,
|
|
23
|
+
):
|
|
24
|
+
request_payload = {
|
|
25
|
+
"date_from": date_from.format("YYYY-MM-DD HH:mm:ss"),
|
|
26
|
+
"date_to": date_to.format("YYYY-MM-DD HH:mm:ss"),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
json_string = json.dumps(request_payload)
|
|
30
|
+
signature = self.generateSignature(json_string)
|
|
31
|
+
headers = {
|
|
32
|
+
"merchant": self.public_key,
|
|
33
|
+
"Signature": signature,
|
|
34
|
+
"Content-Type": "application/json",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
next_page_iterator = None
|
|
38
|
+
url = f"{self.base_url}/{path}"
|
|
39
|
+
|
|
40
|
+
while True:
|
|
41
|
+
payload = request_payload.copy()
|
|
42
|
+
if next_page_iterator:
|
|
43
|
+
payload["page_iterator"] = next_page_iterator
|
|
44
|
+
|
|
45
|
+
response = self.client.post(url, headers=headers, json=payload)
|
|
46
|
+
response.raise_for_status()
|
|
47
|
+
response_json = response.json()
|
|
48
|
+
|
|
49
|
+
if path == "subscriptions":
|
|
50
|
+
data = response_json["subscriptions"]
|
|
51
|
+
for _, value in data.items():
|
|
52
|
+
if "updated_at" in value:
|
|
53
|
+
value["updated_at"] = pendulum.parse(value["updated_at"])
|
|
54
|
+
yield value
|
|
55
|
+
|
|
56
|
+
else:
|
|
57
|
+
data = response_json["orders"]
|
|
58
|
+
for value in data:
|
|
59
|
+
if "updated_at" in value:
|
|
60
|
+
value["updated_at"] = pendulum.parse(value["updated_at"])
|
|
61
|
+
yield value
|
|
62
|
+
|
|
63
|
+
next_page_iterator = response_json.get("metadata", {}).get(
|
|
64
|
+
"next_page_iterator"
|
|
65
|
+
)
|
|
66
|
+
if not next_page_iterator or next_page_iterator == "None":
|
|
67
|
+
break
|
|
68
|
+
|
|
69
|
+
def generateSignature(self, json_string):
|
|
70
|
+
data = self.public_key + json_string + self.public_key
|
|
71
|
+
hmac_hash = hmac.new(
|
|
72
|
+
self.secret_key.encode("utf-8"), data.encode("utf-8"), hashlib.sha512
|
|
73
|
+
).digest()
|
|
74
|
+
return base64.b64encode(hmac_hash.hex().encode("utf-8")).decode("utf-8")
|
ingestr/src/sources.py
CHANGED
|
@@ -388,6 +388,7 @@ class MongoDbSource:
|
|
|
388
388
|
parallel=True,
|
|
389
389
|
incremental=incremental,
|
|
390
390
|
)
|
|
391
|
+
table_instance.max_table_nesting = 1
|
|
391
392
|
|
|
392
393
|
return table_instance
|
|
393
394
|
|
|
@@ -697,46 +698,47 @@ class StripeAnalyticsSource:
|
|
|
697
698
|
if not api_key:
|
|
698
699
|
raise ValueError("api_key in the URI is required to connect to Stripe")
|
|
699
700
|
|
|
700
|
-
|
|
701
|
-
if table == "balancetransaction":
|
|
702
|
-
table = "BalanceTransaction"
|
|
703
|
-
else:
|
|
704
|
-
table = table.capitalize()
|
|
705
|
-
|
|
706
|
-
if table in [
|
|
707
|
-
"Subscription",
|
|
708
|
-
"Account",
|
|
709
|
-
"Coupon",
|
|
710
|
-
"Customer",
|
|
711
|
-
"Product",
|
|
712
|
-
"Price",
|
|
713
|
-
"BalanceTransaction",
|
|
714
|
-
"Invoice",
|
|
715
|
-
"Event",
|
|
716
|
-
"Charge",
|
|
717
|
-
]:
|
|
718
|
-
endpoint = table
|
|
719
|
-
else:
|
|
720
|
-
raise ValueError(
|
|
721
|
-
f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
722
|
-
)
|
|
701
|
+
table = table.lower()
|
|
723
702
|
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
if kwargs.get("interval_end"):
|
|
729
|
-
date_args["end_date"] = kwargs.get("interval_end")
|
|
730
|
-
|
|
731
|
-
from ingestr.src.stripe_analytics import stripe_source
|
|
703
|
+
from ingestr.src.stripe_analytics.settings import (
|
|
704
|
+
ENDPOINTS,
|
|
705
|
+
INCREMENTAL_ENDPOINTS,
|
|
706
|
+
)
|
|
732
707
|
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
708
|
+
if table in ENDPOINTS:
|
|
709
|
+
endpoint = ENDPOINTS[table]
|
|
710
|
+
from ingestr.src.stripe_analytics import stripe_source
|
|
711
|
+
|
|
712
|
+
return stripe_source(
|
|
713
|
+
endpoints=[
|
|
714
|
+
endpoint,
|
|
715
|
+
],
|
|
716
|
+
stripe_secret_key=api_key[0],
|
|
717
|
+
start_date=kwargs.get("interval_start", None),
|
|
718
|
+
end_date=kwargs.get("interval_end", None),
|
|
719
|
+
).with_resources(endpoint)
|
|
720
|
+
|
|
721
|
+
elif table in INCREMENTAL_ENDPOINTS:
|
|
722
|
+
endpoint = INCREMENTAL_ENDPOINTS[table]
|
|
723
|
+
from ingestr.src.stripe_analytics import incremental_stripe_source
|
|
724
|
+
|
|
725
|
+
def nullable_date(date_str: Optional[str]):
|
|
726
|
+
if date_str:
|
|
727
|
+
return ensure_pendulum_datetime(date_str)
|
|
728
|
+
return None
|
|
729
|
+
|
|
730
|
+
return incremental_stripe_source(
|
|
731
|
+
endpoints=[
|
|
732
|
+
endpoint,
|
|
733
|
+
],
|
|
734
|
+
stripe_secret_key=api_key[0],
|
|
735
|
+
initial_start_date=nullable_date(kwargs.get("interval_start", None)),
|
|
736
|
+
end_date=nullable_date(kwargs.get("interval_end", None)),
|
|
737
|
+
).with_resources(endpoint)
|
|
738
|
+
|
|
739
|
+
raise ValueError(
|
|
740
|
+
f"Resource '{table}' is not supported for stripe source yet, if you are interested in it please create a GitHub issue at https://github.com/bruin-data/ingestr"
|
|
741
|
+
)
|
|
740
742
|
|
|
741
743
|
|
|
742
744
|
class FacebookAdsSource:
|
|
@@ -2431,3 +2433,76 @@ class AttioSource:
|
|
|
2431
2433
|
)
|
|
2432
2434
|
except ResourcesNotFoundError:
|
|
2433
2435
|
raise UnsupportedResourceError(table_name, "Attio")
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
class SmartsheetSource:
|
|
2439
|
+
def handles_incrementality(self) -> bool:
|
|
2440
|
+
return False
|
|
2441
|
+
|
|
2442
|
+
# smartsheet://?access_token=<access_token>
|
|
2443
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2444
|
+
if kwargs.get("incremental_key"):
|
|
2445
|
+
raise ValueError("Incremental loads are not supported for Smartsheet")
|
|
2446
|
+
|
|
2447
|
+
if not table:
|
|
2448
|
+
raise ValueError(
|
|
2449
|
+
"Source table (sheet_id) is required to connect to Smartsheet"
|
|
2450
|
+
)
|
|
2451
|
+
|
|
2452
|
+
source_parts = urlparse(uri)
|
|
2453
|
+
source_fields = parse_qs(source_parts.query)
|
|
2454
|
+
access_token = source_fields.get("access_token")
|
|
2455
|
+
|
|
2456
|
+
if not access_token:
|
|
2457
|
+
raise ValueError(
|
|
2458
|
+
"access_token in the URI is required to connect to Smartsheet"
|
|
2459
|
+
)
|
|
2460
|
+
|
|
2461
|
+
from ingestr.src.smartsheets import smartsheet_source
|
|
2462
|
+
|
|
2463
|
+
return smartsheet_source(
|
|
2464
|
+
access_token=access_token[0],
|
|
2465
|
+
sheet_id=table, # table is now a single sheet_id
|
|
2466
|
+
)
|
|
2467
|
+
|
|
2468
|
+
|
|
2469
|
+
class SolidgateSource:
|
|
2470
|
+
def handles_incrementality(self) -> bool:
|
|
2471
|
+
return True
|
|
2472
|
+
|
|
2473
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2474
|
+
parsed_uri = urlparse(uri)
|
|
2475
|
+
query_params = parse_qs(parsed_uri.query)
|
|
2476
|
+
public_key = query_params.get("public_key")
|
|
2477
|
+
secret_key = query_params.get("secret_key")
|
|
2478
|
+
|
|
2479
|
+
if public_key is None:
|
|
2480
|
+
raise MissingValueError("public_key", "Solidgate")
|
|
2481
|
+
|
|
2482
|
+
if secret_key is None:
|
|
2483
|
+
raise MissingValueError("secret_key", "Solidgate")
|
|
2484
|
+
|
|
2485
|
+
table_name = table.replace(" ", "")
|
|
2486
|
+
|
|
2487
|
+
start_date = kwargs.get("interval_start")
|
|
2488
|
+
if start_date is None:
|
|
2489
|
+
start_date = pendulum.yesterday().in_tz("UTC")
|
|
2490
|
+
else:
|
|
2491
|
+
start_date = ensure_pendulum_datetime(start_date).in_tz("UTC")
|
|
2492
|
+
|
|
2493
|
+
end_date = kwargs.get("interval_end")
|
|
2494
|
+
|
|
2495
|
+
if end_date is not None:
|
|
2496
|
+
end_date = ensure_pendulum_datetime(end_date).in_tz("UTC")
|
|
2497
|
+
|
|
2498
|
+
from ingestr.src.solidgate import solidgate_source
|
|
2499
|
+
|
|
2500
|
+
try:
|
|
2501
|
+
return solidgate_source(
|
|
2502
|
+
public_key=public_key[0],
|
|
2503
|
+
secret_key=secret_key[0],
|
|
2504
|
+
start_date=start_date,
|
|
2505
|
+
end_date=end_date,
|
|
2506
|
+
).with_resources(table_name)
|
|
2507
|
+
except ResourcesNotFoundError:
|
|
2508
|
+
raise UnsupportedResourceError(table_name, "Solidgate")
|
|
@@ -8,12 +8,11 @@ from dlt.sources import DltResource
|
|
|
8
8
|
from pendulum import DateTime
|
|
9
9
|
|
|
10
10
|
from .helpers import pagination, transform_date
|
|
11
|
-
from .settings import ENDPOINTS, INCREMENTAL_ENDPOINTS
|
|
12
11
|
|
|
13
12
|
|
|
14
13
|
@dlt.source(max_table_nesting=0)
|
|
15
14
|
def stripe_source(
|
|
16
|
-
endpoints: Tuple[str, ...]
|
|
15
|
+
endpoints: Tuple[str, ...],
|
|
17
16
|
stripe_secret_key: str = dlt.secrets.value,
|
|
18
17
|
start_date: Optional[DateTime] = None,
|
|
19
18
|
end_date: Optional[DateTime] = None,
|
|
@@ -53,7 +52,7 @@ def stripe_source(
|
|
|
53
52
|
|
|
54
53
|
@dlt.source
|
|
55
54
|
def incremental_stripe_source(
|
|
56
|
-
endpoints: Tuple[str, ...]
|
|
55
|
+
endpoints: Tuple[str, ...],
|
|
57
56
|
stripe_secret_key: str = dlt.secrets.value,
|
|
58
57
|
initial_start_date: Optional[DateTime] = None,
|
|
59
58
|
end_date: Optional[DateTime] = None,
|
|
@@ -2,13 +2,25 @@
|
|
|
2
2
|
|
|
3
3
|
# the most popular endpoints
|
|
4
4
|
# Full list of the Stripe API endpoints you can find here: https://stripe.com/docs/api.
|
|
5
|
-
ENDPOINTS =
|
|
6
|
-
"Subscription",
|
|
7
|
-
"Account",
|
|
8
|
-
"Coupon",
|
|
9
|
-
"Customer",
|
|
10
|
-
"Product",
|
|
11
|
-
"Price",
|
|
12
|
-
|
|
5
|
+
ENDPOINTS = {
|
|
6
|
+
"subscription": "Subscription",
|
|
7
|
+
"account": "Account",
|
|
8
|
+
"coupon": "Coupon",
|
|
9
|
+
"customer": "Customer",
|
|
10
|
+
"product": "Product",
|
|
11
|
+
"price": "Price",
|
|
12
|
+
"shippingrate": "ShippingRate",
|
|
13
|
+
"dispute": "Dispute",
|
|
14
|
+
"subscriptionitem": "SubscriptionItem",
|
|
15
|
+
"checkoutsession": "CheckoutSession",
|
|
16
|
+
}
|
|
13
17
|
# possible incremental endpoints
|
|
14
|
-
INCREMENTAL_ENDPOINTS =
|
|
18
|
+
INCREMENTAL_ENDPOINTS = {
|
|
19
|
+
"event": "Event",
|
|
20
|
+
"invoice": "Invoice",
|
|
21
|
+
"balancetransaction": "BalanceTransaction",
|
|
22
|
+
"charge": "Charge",
|
|
23
|
+
"applicationfee": "ApplicationFee",
|
|
24
|
+
"setupattempt": "SetupAttempt",
|
|
25
|
+
"creditnote": "CreditNote",
|
|
26
|
+
}
|
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
import sys
|
|
2
|
+
import unittest
|
|
3
|
+
from unittest.mock import MagicMock, patch
|
|
4
|
+
|
|
5
|
+
import smartsheet # type: ignore
|
|
6
|
+
from smartsheet.models import Cell, Column, Row, Sheet # type: ignore
|
|
7
|
+
|
|
8
|
+
from ingestr.src.smartsheets import _get_sheet_data, smartsheet_source
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def pp(x):
|
|
12
|
+
print(x, file=sys.stderr)
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class TestSmartsheetSource(unittest.TestCase):
|
|
16
|
+
@patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
|
|
17
|
+
def test_smartsheet_source_success(self, mock_smartsheet_client):
|
|
18
|
+
# Mock Smartsheet client and its methods
|
|
19
|
+
mock_client_instance = mock_smartsheet_client.return_value
|
|
20
|
+
|
|
21
|
+
# Mock sheet details response
|
|
22
|
+
mock_sheet_details = Sheet(
|
|
23
|
+
{
|
|
24
|
+
"id": 123,
|
|
25
|
+
"name": "Test Sheet 1",
|
|
26
|
+
"columns": [
|
|
27
|
+
Column(
|
|
28
|
+
{"id": 1, "title": "Col A", "type": "TEXT_NUMBER", "index": 0}
|
|
29
|
+
),
|
|
30
|
+
Column(
|
|
31
|
+
{"id": 2, "title": "Col B", "type": "TEXT_NUMBER", "index": 1}
|
|
32
|
+
),
|
|
33
|
+
],
|
|
34
|
+
"rows": [
|
|
35
|
+
Row(
|
|
36
|
+
{
|
|
37
|
+
"id": 101,
|
|
38
|
+
"sheetId": 123,
|
|
39
|
+
"cells": [
|
|
40
|
+
Cell({"columnId": 1, "value": "r1c1"}),
|
|
41
|
+
Cell({"columnId": 2, "value": "r1c2"}),
|
|
42
|
+
],
|
|
43
|
+
}
|
|
44
|
+
),
|
|
45
|
+
Row(
|
|
46
|
+
{
|
|
47
|
+
"id": 102,
|
|
48
|
+
"sheetId": 123,
|
|
49
|
+
"cells": [
|
|
50
|
+
Cell({"columnId": 1, "value": "r2c1"}),
|
|
51
|
+
Cell({"columnId": 2, "value": "r2c2"}),
|
|
52
|
+
],
|
|
53
|
+
}
|
|
54
|
+
),
|
|
55
|
+
],
|
|
56
|
+
}
|
|
57
|
+
)
|
|
58
|
+
mock_client_instance.Sheets.get_sheet.return_value = mock_sheet_details
|
|
59
|
+
|
|
60
|
+
resource = smartsheet_source(access_token="test_token", sheet_id="123")
|
|
61
|
+
data = list(resource)
|
|
62
|
+
self.assertEqual(len(data), 2)
|
|
63
|
+
self.assertEqual(data[0], {"Col A": "r1c1", "Col B": "r1c2"})
|
|
64
|
+
self.assertEqual(data[1], {"Col A": "r2c1", "Col B": "r2c2"})
|
|
65
|
+
|
|
66
|
+
mock_smartsheet_client.assert_called_once_with("test_token")
|
|
67
|
+
mock_client_instance.Sheets.get_sheet.assert_any_call(
|
|
68
|
+
123, include=["objectValue"]
|
|
69
|
+
) # for resource name
|
|
70
|
+
mock_client_instance.Sheets.get_sheet.assert_any_call(
|
|
71
|
+
123
|
|
72
|
+
) # for _get_sheet_data
|
|
73
|
+
|
|
74
|
+
@patch("ingestr.src.smartsheets.smartsheet.Smartsheet")
|
|
75
|
+
def test_smartsheet_source_api_error(self, mock_smartsheet_client):
|
|
76
|
+
mock_client_instance = mock_smartsheet_client.return_value
|
|
77
|
+
mock_client_instance.Sheets.get_sheet.side_effect = (
|
|
78
|
+
smartsheet.exceptions.ApiError("API Error", 500)
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
with self.assertRaises(smartsheet.exceptions.ApiError):
|
|
82
|
+
source = smartsheet_source(access_token="test_token", sheet_id="123")
|
|
83
|
+
# Consume the generator to trigger the API call
|
|
84
|
+
list(source)
|
|
85
|
+
|
|
86
|
+
def test_get_sheet_data(self):
|
|
87
|
+
mock_smartsheet_client_instance = MagicMock()
|
|
88
|
+
mock_sheet = Sheet(
|
|
89
|
+
{
|
|
90
|
+
"id": 456,
|
|
91
|
+
"name": "Data Sheet",
|
|
92
|
+
"columns": [
|
|
93
|
+
Column(
|
|
94
|
+
{"id": 10, "title": "ID", "type": "TEXT_NUMBER", "index": 0}
|
|
95
|
+
),
|
|
96
|
+
Column(
|
|
97
|
+
{"id": 20, "title": "Value", "type": "TEXT_NUMBER", "index": 1}
|
|
98
|
+
),
|
|
99
|
+
],
|
|
100
|
+
"rows": [
|
|
101
|
+
Row(
|
|
102
|
+
{
|
|
103
|
+
"id": 201,
|
|
104
|
+
"sheetId": 456,
|
|
105
|
+
"cells": [
|
|
106
|
+
Cell({"columnId": 10, "value": 1}),
|
|
107
|
+
Cell({"columnId": 20, "value": "Alpha"}),
|
|
108
|
+
],
|
|
109
|
+
}
|
|
110
|
+
),
|
|
111
|
+
Row(
|
|
112
|
+
{
|
|
113
|
+
"id": 202,
|
|
114
|
+
"sheetId": 456,
|
|
115
|
+
"cells": [
|
|
116
|
+
Cell({"columnId": 10, "value": 2}),
|
|
117
|
+
Cell({"columnId": 20, "value": "Beta"}),
|
|
118
|
+
],
|
|
119
|
+
}
|
|
120
|
+
),
|
|
121
|
+
],
|
|
122
|
+
}
|
|
123
|
+
)
|
|
124
|
+
mock_smartsheet_client_instance.Sheets.get_sheet.return_value = mock_sheet
|
|
125
|
+
|
|
126
|
+
data_generator = _get_sheet_data(mock_smartsheet_client_instance, 456)
|
|
127
|
+
data = list(data_generator)
|
|
128
|
+
|
|
129
|
+
self.assertEqual(len(data), 2)
|
|
130
|
+
self.assertEqual(data[0], {"ID": 1, "Value": "Alpha"})
|
|
131
|
+
self.assertEqual(data[1], {"ID": 2, "Value": "Beta"})
|
|
132
|
+
mock_smartsheet_client_instance.Sheets.get_sheet.assert_called_once_with(456)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
if __name__ == "__main__":
|
|
136
|
+
unittest.main()
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.49
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -163,6 +163,7 @@ Requires-Dist: shellingham==1.5.4
|
|
|
163
163
|
Requires-Dist: simple-salesforce==1.12.6
|
|
164
164
|
Requires-Dist: simplejson==3.20.1
|
|
165
165
|
Requires-Dist: six==1.17.0
|
|
166
|
+
Requires-Dist: smartsheet-python-sdk==3.0.5
|
|
166
167
|
Requires-Dist: smmap==5.0.2
|
|
167
168
|
Requires-Dist: snowflake-connector-python==3.14.0
|
|
168
169
|
Requires-Dist: snowflake-sqlalchemy==1.6.1
|
|
@@ -467,6 +468,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
467
468
|
<td>✅</td>
|
|
468
469
|
<td>-</td>
|
|
469
470
|
</tr>
|
|
471
|
+
<tr>
|
|
472
|
+
<td>Smartsheet</td>
|
|
473
|
+
<td>✅</td>
|
|
474
|
+
<td>-</td>
|
|
475
|
+
</tr>
|
|
470
476
|
<tr>
|
|
471
477
|
<td>Stripe</td>
|
|
472
478
|
<td>✅</td>
|
|
@@ -1,17 +1,17 @@
|
|
|
1
1
|
ingestr/conftest.py,sha256=Q03FIJIZpLBbpj55cfCHIKEjc1FCvWJhMF2cidUJKQU,1748
|
|
2
|
-
ingestr/main.py,sha256=
|
|
2
|
+
ingestr/main.py,sha256=rHxHQAbd0ccW2e2kQSWlv7-5qcc2ZB6Eh3vyjm4Nzns,25550
|
|
3
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
4
4
|
ingestr/src/blob.py,sha256=onMe5ZHxPXTdcB_s2oGNdMo-XQJ3ajwOsWE9eSTGFmc,1495
|
|
5
|
-
ingestr/src/buildinfo.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=Q38dT9-xAEHMT3iRCHcD0SOhlVbTR8sRnjHj5TYEUfo,21
|
|
6
6
|
ingestr/src/destinations.py,sha256=41Bj1UgxR8a2KcZWqtGw74AKZKnSBrueQRnBdrf3c-A,16003
|
|
7
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
8
|
-
ingestr/src/factory.py,sha256=
|
|
8
|
+
ingestr/src/factory.py,sha256=FXWJLFfBsKMzUwtsyaaruZU-_OLFKivobj6Olse9vSI,5741
|
|
9
9
|
ingestr/src/filters.py,sha256=C-_TIVkF_cxZBgG-Run2Oyn0TAhJgA8IWXZ-OPY3uek,1136
|
|
10
10
|
ingestr/src/http_client.py,sha256=UwPiv95EfHPdT4525xeLFJ1AYlf-cyaHKRU-2QnZt2o,435
|
|
11
11
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
12
12
|
ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
|
|
13
|
-
ingestr/src/resource.py,sha256=
|
|
14
|
-
ingestr/src/sources.py,sha256=
|
|
13
|
+
ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
|
|
14
|
+
ingestr/src/sources.py,sha256=SnFxil6gU5RIYiSUfE6Upq2se0YeRSuTIKK3Jxn-YKI,86587
|
|
15
15
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
16
16
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
17
17
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
@@ -82,7 +82,7 @@ ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYH
|
|
|
82
82
|
ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
|
|
83
83
|
ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
|
|
84
84
|
ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
|
|
85
|
-
ingestr/src/mongodb/__init__.py,sha256=
|
|
85
|
+
ingestr/src/mongodb/__init__.py,sha256=T-RYPS_skl_2gNVfYWWXan2bVQYmm0bFBcCCqG5ejvg,7275
|
|
86
86
|
ingestr/src/mongodb/helpers.py,sha256=H0GpOK3bPBhFWBEhJZOjywUBdzih6MOpmyVO_cKSN14,24178
|
|
87
87
|
ingestr/src/notion/__init__.py,sha256=36wUui8finbc85ObkRMq8boMraXMUehdABN_AMe_hzA,1834
|
|
88
88
|
ingestr/src/notion/settings.py,sha256=MwQVZViJtnvOegfjXYc_pJ50oUYgSRPgwqu7TvpeMOA,82
|
|
@@ -108,11 +108,14 @@ ingestr/src/shopify/settings.py,sha256=StY0EPr7wFJ7KzRRDN4TKxV0_gkIS1wPj2eR4AYSs
|
|
|
108
108
|
ingestr/src/slack/__init__.py,sha256=pyDukxcilqTAe_bBzfWJ8Vxi83S-XEdEFBH2pEgILrM,10113
|
|
109
109
|
ingestr/src/slack/helpers.py,sha256=08TLK7vhFvH_uekdLVOLF3bTDe1zgH0QxHObXHzk1a8,6545
|
|
110
110
|
ingestr/src/slack/settings.py,sha256=NhKn4y1zokEa5EmIZ05wtj_-I0GOASXZ5V81M1zXCtY,457
|
|
111
|
+
ingestr/src/smartsheets/__init__.py,sha256=pdzSV7rA0XYD5Xa1u4zb6vziy5iFXIQNROkpJ9oYas0,1623
|
|
112
|
+
ingestr/src/solidgate/__init__.py,sha256=vpoXu0Ox3zE_WPSzdsA6iUG1_XBa9OaA5F7eFBbZYuQ,2819
|
|
113
|
+
ingestr/src/solidgate/helpers.py,sha256=_PuHmKZ-jpNEsPxRgXCzu39PsaygVDCpnoMTZAYSpHE,2432
|
|
111
114
|
ingestr/src/sql_database/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
112
115
|
ingestr/src/sql_database/callbacks.py,sha256=sEFFmXxAURY3yeBjnawigDtq9LBCvi8HFqG4kLd7tMU,2002
|
|
113
|
-
ingestr/src/stripe_analytics/__init__.py,sha256=
|
|
116
|
+
ingestr/src/stripe_analytics/__init__.py,sha256=FBkZu5op5Z-FceEi4zG7qcAgZfUYJRPMVPPrPMjvmXw,4502
|
|
114
117
|
ingestr/src/stripe_analytics/helpers.py,sha256=iqZOyiGIOhOAhVXXU16DP0hkkTKcTrDu69vAJoTxgEo,1976
|
|
115
|
-
ingestr/src/stripe_analytics/settings.py,sha256=
|
|
118
|
+
ingestr/src/stripe_analytics/settings.py,sha256=fA2j_6FquEmyRyB799P4SncwLwK1S1u9WFNjbzu91kY,786
|
|
116
119
|
ingestr/src/telemetry/event.py,sha256=W7bs4uVfPakQ5otmiqgqu1l5SqjYx1p87wudnWXckBc,949
|
|
117
120
|
ingestr/src/testdata/fakebqcredentials.json,sha256=scc6TUc963KAbKTLZCfcmqVzbtzDCW1_8JNRnyAXyy8,628
|
|
118
121
|
ingestr/src/tiktok_ads/__init__.py,sha256=aEqCl3dTH6_d43s1jgAeG1UasEls_SlorORulYMwIL8,4590
|
|
@@ -131,8 +134,9 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
131
134
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
132
135
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
133
136
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
134
|
-
ingestr
|
|
135
|
-
ingestr-0.13.
|
|
136
|
-
ingestr-0.13.
|
|
137
|
-
ingestr-0.13.
|
|
138
|
-
ingestr-0.13.
|
|
137
|
+
ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
|
|
138
|
+
ingestr-0.13.49.dist-info/METADATA,sha256=pkrtX6z_Vkjc4GnSTW_6SmskX8MQm_ASZOX6hdFlJgM,13983
|
|
139
|
+
ingestr-0.13.49.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
140
|
+
ingestr-0.13.49.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
141
|
+
ingestr-0.13.49.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
142
|
+
ingestr-0.13.49.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|