ingestr 0.13.10__py3-none-any.whl → 0.13.11__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/src/applovin/__init__.py +8 -7
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/factory.py +2 -0
- ingestr/src/salesforce/__init__.py +149 -0
- ingestr/src/salesforce/helpers.py +64 -0
- ingestr/src/sources.py +71 -6
- {ingestr-0.13.10.dist-info → ingestr-0.13.11.dist-info}/METADATA +17 -11
- {ingestr-0.13.10.dist-info → ingestr-0.13.11.dist-info}/RECORD +11 -9
- {ingestr-0.13.10.dist-info → ingestr-0.13.11.dist-info}/WHEEL +0 -0
- {ingestr-0.13.10.dist-info → ingestr-0.13.11.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.10.dist-info → ingestr-0.13.11.dist-info}/licenses/LICENSE.md +0 -0
ingestr/src/applovin/__init__.py
CHANGED
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
from datetime import datetime,
|
|
1
|
+
from datetime import datetime, timedelta, timezone
|
|
2
2
|
from enum import Enum
|
|
3
3
|
from typing import Dict, List, Optional
|
|
4
|
-
from requests import Response
|
|
5
4
|
|
|
6
5
|
import dlt
|
|
7
6
|
from dlt.sources.rest_api import EndpointResource, RESTAPIConfig, rest_api_resources
|
|
7
|
+
from requests import Response
|
|
8
8
|
|
|
9
9
|
|
|
10
10
|
class InvalidCustomReportError(Exception):
|
|
@@ -13,9 +13,11 @@ class InvalidCustomReportError(Exception):
|
|
|
13
13
|
"Custom report should be in the format 'custom:{endpoint}:{report_type}:{dimensions}"
|
|
14
14
|
)
|
|
15
15
|
|
|
16
|
+
|
|
16
17
|
class ClientError(Exception):
|
|
17
18
|
pass
|
|
18
19
|
|
|
20
|
+
|
|
19
21
|
TYPE_HINTS = {
|
|
20
22
|
"application_is_hidden": {"data_type": "bool"},
|
|
21
23
|
"average_cpa": {"data_type": "double"},
|
|
@@ -119,7 +121,6 @@ def applovin_source(
|
|
|
119
121
|
end_date: Optional[str],
|
|
120
122
|
custom: Optional[str],
|
|
121
123
|
):
|
|
122
|
-
|
|
123
124
|
backfill = False
|
|
124
125
|
if end_date is None:
|
|
125
126
|
backfill = True
|
|
@@ -127,7 +128,7 @@ def applovin_source(
|
|
|
127
128
|
# use the greatest of yesterday and start_date
|
|
128
129
|
end_date = max(
|
|
129
130
|
datetime.now(timezone.utc) - timedelta(days=1),
|
|
130
|
-
datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc)
|
|
131
|
+
datetime.fromisoformat(start_date).replace(tzinfo=timezone.utc),
|
|
131
132
|
).strftime("%Y-%m-%d")
|
|
132
133
|
|
|
133
134
|
config: RESTAPIConfig = {
|
|
@@ -157,7 +158,7 @@ def applovin_source(
|
|
|
157
158
|
"paginator": "single_page",
|
|
158
159
|
"response_actions": [
|
|
159
160
|
http_error_handler,
|
|
160
|
-
]
|
|
161
|
+
],
|
|
161
162
|
},
|
|
162
163
|
},
|
|
163
164
|
"resources": [
|
|
@@ -177,8 +178,7 @@ def applovin_source(
|
|
|
177
178
|
"advertiser-probabilistic-report",
|
|
178
179
|
"probabilisticReport",
|
|
179
180
|
exclude(
|
|
180
|
-
REPORT_SCHEMA[ReportType.ADVERTISER],
|
|
181
|
-
PROBABILISTIC_REPORT_EXCLUDE
|
|
181
|
+
REPORT_SCHEMA[ReportType.ADVERTISER], PROBABILISTIC_REPORT_EXCLUDE
|
|
182
182
|
),
|
|
183
183
|
ReportType.ADVERTISER,
|
|
184
184
|
),
|
|
@@ -256,6 +256,7 @@ def exclude(source: List[str], exclude_list: List[str]) -> List[str]:
|
|
|
256
256
|
def build_type_hints(cols: List[str]) -> dict:
|
|
257
257
|
return {col: TYPE_HINTS[col] for col in cols if col in TYPE_HINTS}
|
|
258
258
|
|
|
259
|
+
|
|
259
260
|
def http_error_handler(resp: Response):
|
|
260
261
|
if not resp.ok:
|
|
261
262
|
raise ClientError(f"HTTP Status {resp.status_code}: {resp.text}")
|
ingestr/src/buildinfo.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = "v0.13.
|
|
1
|
+
version = "v0.13.11"
|
ingestr/src/factory.py
CHANGED
|
@@ -42,6 +42,7 @@ from ingestr.src.sources import (
|
|
|
42
42
|
MongoDbSource,
|
|
43
43
|
NotionSource,
|
|
44
44
|
S3Source,
|
|
45
|
+
SalesforceSource,
|
|
45
46
|
ShopifySource,
|
|
46
47
|
SlackSource,
|
|
47
48
|
SqlSource,
|
|
@@ -137,6 +138,7 @@ class SourceDestinationFactory:
|
|
|
137
138
|
"linkedinads": LinkedInAdsSource,
|
|
138
139
|
"applovin": AppLovinSource,
|
|
139
140
|
"applovinmax": ApplovinMaxSource,
|
|
141
|
+
"salesforce": SalesforceSource,
|
|
140
142
|
"personio": PersonioSource,
|
|
141
143
|
}
|
|
142
144
|
destinations: Dict[str, Type[DestinationProtocol]] = {
|
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
from typing import Iterable
|
|
2
|
+
|
|
3
|
+
import dlt
|
|
4
|
+
from dlt.common.typing import TDataItem
|
|
5
|
+
from dlt.sources import DltResource, incremental
|
|
6
|
+
from simple_salesforce import Salesforce
|
|
7
|
+
|
|
8
|
+
from .helpers import get_records
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
@dlt.source(name="salesforce")
|
|
12
|
+
def salesforce_source(
|
|
13
|
+
username: str,
|
|
14
|
+
password: str,
|
|
15
|
+
token: str,
|
|
16
|
+
) -> Iterable[DltResource]:
|
|
17
|
+
"""
|
|
18
|
+
Retrieves data from Salesforce using the Salesforce API.
|
|
19
|
+
|
|
20
|
+
Args:
|
|
21
|
+
username (str): The username for authentication.
|
|
22
|
+
password (str): The password for authentication.
|
|
23
|
+
token (str): The security token for authentication.
|
|
24
|
+
|
|
25
|
+
Yields:
|
|
26
|
+
DltResource: Data resources from Salesforce.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
client = Salesforce(username, password, token)
|
|
30
|
+
|
|
31
|
+
# define resources
|
|
32
|
+
@dlt.resource(write_disposition="replace")
|
|
33
|
+
def user() -> Iterable[TDataItem]:
|
|
34
|
+
yield get_records(client, "User")
|
|
35
|
+
|
|
36
|
+
@dlt.resource(write_disposition="replace")
|
|
37
|
+
def user_role() -> Iterable[TDataItem]:
|
|
38
|
+
yield get_records(client, "UserRole")
|
|
39
|
+
|
|
40
|
+
@dlt.resource(write_disposition="merge")
|
|
41
|
+
def opportunity(
|
|
42
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
43
|
+
"SystemModstamp", initial_value=None
|
|
44
|
+
),
|
|
45
|
+
) -> Iterable[TDataItem]:
|
|
46
|
+
yield get_records(
|
|
47
|
+
client, "Opportunity", last_timestamp.last_value, "SystemModstamp"
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
@dlt.resource(write_disposition="merge")
|
|
51
|
+
def opportunity_line_item(
|
|
52
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
53
|
+
"SystemModstamp", initial_value=None
|
|
54
|
+
),
|
|
55
|
+
) -> Iterable[TDataItem]:
|
|
56
|
+
yield get_records(
|
|
57
|
+
client, "OpportunityLineItem", last_timestamp.last_value, "SystemModstamp"
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
@dlt.resource(write_disposition="merge")
|
|
61
|
+
def opportunity_contact_role(
|
|
62
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
63
|
+
"SystemModstamp", initial_value=None
|
|
64
|
+
),
|
|
65
|
+
) -> Iterable[TDataItem]:
|
|
66
|
+
yield get_records(
|
|
67
|
+
client,
|
|
68
|
+
"OpportunityContactRole",
|
|
69
|
+
last_timestamp.last_value,
|
|
70
|
+
"SystemModstamp",
|
|
71
|
+
)
|
|
72
|
+
|
|
73
|
+
@dlt.resource(write_disposition="merge")
|
|
74
|
+
def account(
|
|
75
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
76
|
+
"LastModifiedDate", initial_value=None
|
|
77
|
+
),
|
|
78
|
+
) -> Iterable[TDataItem]:
|
|
79
|
+
yield get_records(
|
|
80
|
+
client, "Account", last_timestamp.last_value, "LastModifiedDate"
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
@dlt.resource(write_disposition="replace")
|
|
84
|
+
def contact() -> Iterable[TDataItem]:
|
|
85
|
+
yield get_records(client, "Contact")
|
|
86
|
+
|
|
87
|
+
@dlt.resource(write_disposition="replace")
|
|
88
|
+
def lead() -> Iterable[TDataItem]:
|
|
89
|
+
yield get_records(client, "Lead")
|
|
90
|
+
|
|
91
|
+
@dlt.resource(write_disposition="replace")
|
|
92
|
+
def campaign() -> Iterable[TDataItem]:
|
|
93
|
+
yield get_records(client, "Campaign")
|
|
94
|
+
|
|
95
|
+
@dlt.resource(write_disposition="merge")
|
|
96
|
+
def campaign_member(
|
|
97
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
98
|
+
"SystemModstamp", initial_value=None
|
|
99
|
+
),
|
|
100
|
+
) -> Iterable[TDataItem]:
|
|
101
|
+
yield get_records(
|
|
102
|
+
client, "CampaignMember", last_timestamp.last_value, "SystemModstamp"
|
|
103
|
+
)
|
|
104
|
+
|
|
105
|
+
@dlt.resource(write_disposition="replace")
|
|
106
|
+
def product() -> Iterable[TDataItem]:
|
|
107
|
+
yield get_records(client, "Product2")
|
|
108
|
+
|
|
109
|
+
@dlt.resource(write_disposition="replace")
|
|
110
|
+
def pricebook() -> Iterable[TDataItem]:
|
|
111
|
+
yield get_records(client, "Pricebook2")
|
|
112
|
+
|
|
113
|
+
@dlt.resource(write_disposition="replace")
|
|
114
|
+
def pricebook_entry() -> Iterable[TDataItem]:
|
|
115
|
+
yield get_records(client, "PricebookEntry")
|
|
116
|
+
|
|
117
|
+
@dlt.resource(write_disposition="merge")
|
|
118
|
+
def task(
|
|
119
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
120
|
+
"SystemModstamp", initial_value=None
|
|
121
|
+
),
|
|
122
|
+
) -> Iterable[TDataItem]:
|
|
123
|
+
yield get_records(client, "Task", last_timestamp.last_value, "SystemModstamp")
|
|
124
|
+
|
|
125
|
+
@dlt.resource(write_disposition="merge")
|
|
126
|
+
def event(
|
|
127
|
+
last_timestamp: incremental[str] = dlt.sources.incremental(
|
|
128
|
+
"SystemModstamp", initial_value=None
|
|
129
|
+
),
|
|
130
|
+
) -> Iterable[TDataItem]:
|
|
131
|
+
yield get_records(client, "Event", last_timestamp.last_value, "SystemModstamp")
|
|
132
|
+
|
|
133
|
+
return (
|
|
134
|
+
user,
|
|
135
|
+
user_role,
|
|
136
|
+
opportunity,
|
|
137
|
+
opportunity_line_item,
|
|
138
|
+
opportunity_contact_role,
|
|
139
|
+
account,
|
|
140
|
+
contact,
|
|
141
|
+
lead,
|
|
142
|
+
campaign,
|
|
143
|
+
campaign_member,
|
|
144
|
+
product,
|
|
145
|
+
pricebook,
|
|
146
|
+
pricebook_entry,
|
|
147
|
+
task,
|
|
148
|
+
event,
|
|
149
|
+
)
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Salesforce source helpers"""
|
|
2
|
+
|
|
3
|
+
from typing import Iterable, Optional
|
|
4
|
+
|
|
5
|
+
import pendulum
|
|
6
|
+
from dlt.common.typing import TDataItem
|
|
7
|
+
from simple_salesforce import Salesforce
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def get_records(
|
|
11
|
+
sf: Salesforce,
|
|
12
|
+
sobject: str,
|
|
13
|
+
last_state: Optional[str] = None,
|
|
14
|
+
replication_key: Optional[str] = None,
|
|
15
|
+
) -> Iterable[TDataItem]:
|
|
16
|
+
"""
|
|
17
|
+
Retrieves records from Salesforce for a specified sObject.
|
|
18
|
+
|
|
19
|
+
Args:
|
|
20
|
+
sf (Salesforce): An instance of the Salesforce API client.
|
|
21
|
+
sobject (str): The name of the sObject to retrieve records from.
|
|
22
|
+
last_state (str, optional): The last known state for incremental loading. Defaults to None.
|
|
23
|
+
replication_key (str, optional): The replication key for incremental loading. Defaults to None.
|
|
24
|
+
|
|
25
|
+
Yields:
|
|
26
|
+
Dict[TDataItem]: A dictionary representing a record from the Salesforce sObject.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
# Get all fields for the sobject
|
|
30
|
+
desc = getattr(sf, sobject).describe()
|
|
31
|
+
# Salesforce returns compound fields as separate fields, so we need to filter them out
|
|
32
|
+
compound_fields = {
|
|
33
|
+
f["compoundFieldName"]
|
|
34
|
+
for f in desc["fields"]
|
|
35
|
+
if f["compoundFieldName"] is not None
|
|
36
|
+
} - {"Name"}
|
|
37
|
+
# Salesforce returns datetime fields as timestamps, so we need to convert them
|
|
38
|
+
date_fields = {
|
|
39
|
+
f["name"] for f in desc["fields"] if f["type"] in ("datetime",) and f["name"]
|
|
40
|
+
}
|
|
41
|
+
# If no fields are specified, use all fields except compound fields
|
|
42
|
+
fields = [f["name"] for f in desc["fields"] if f["name"] not in compound_fields]
|
|
43
|
+
|
|
44
|
+
# Generate a predicate to filter records by the replication key
|
|
45
|
+
predicate, order_by, n_records = "", "", 0
|
|
46
|
+
if replication_key:
|
|
47
|
+
if last_state:
|
|
48
|
+
predicate = f"WHERE {replication_key} > {last_state}"
|
|
49
|
+
order_by = f"ORDER BY {replication_key} ASC"
|
|
50
|
+
query = f"SELECT {', '.join(fields)} FROM {sobject} {predicate} {order_by}"
|
|
51
|
+
|
|
52
|
+
# Query all records in batches
|
|
53
|
+
for page in getattr(sf.bulk, sobject).query_all(query, lazy_operation=True):
|
|
54
|
+
for record in page:
|
|
55
|
+
# Strip out the attributes field
|
|
56
|
+
record.pop("attributes", None)
|
|
57
|
+
for field in date_fields:
|
|
58
|
+
# Convert Salesforce timestamps to ISO 8601
|
|
59
|
+
if record.get(field):
|
|
60
|
+
record[field] = pendulum.from_timestamp(
|
|
61
|
+
record[field] / 1000,
|
|
62
|
+
).strftime("%Y-%m-%dT%H:%M:%S.%fZ")
|
|
63
|
+
yield from page
|
|
64
|
+
n_records += len(page)
|
ingestr/src/sources.py
CHANGED
|
@@ -15,7 +15,7 @@ from typing import (
|
|
|
15
15
|
Optional,
|
|
16
16
|
Union,
|
|
17
17
|
)
|
|
18
|
-
from urllib.parse import ParseResult, parse_qs, quote, urlparse
|
|
18
|
+
from urllib.parse import ParseResult, parse_qs, quote, urlencode, urlparse
|
|
19
19
|
|
|
20
20
|
import dlt
|
|
21
21
|
import gcsfs # type: ignore
|
|
@@ -84,6 +84,7 @@ from ingestr.src.linkedin_ads.dimension_time_enum import (
|
|
|
84
84
|
from ingestr.src.mongodb import mongodb_collection
|
|
85
85
|
from ingestr.src.notion import notion_databases
|
|
86
86
|
from ingestr.src.personio import personio_source
|
|
87
|
+
from ingestr.src.salesforce import salesforce_source
|
|
87
88
|
from ingestr.src.shopify import shopify_source
|
|
88
89
|
from ingestr.src.slack import slack_source
|
|
89
90
|
from ingestr.src.sql_database.callbacks import (
|
|
@@ -135,10 +136,46 @@ class SqlSource:
|
|
|
135
136
|
if uri.startswith("mysql://"):
|
|
136
137
|
uri = uri.replace("mysql://", "mysql+pymysql://")
|
|
137
138
|
|
|
139
|
+
# clickhouse://<username>:<password>@<host>:<port>?secure=<secure>
|
|
138
140
|
if uri.startswith("clickhouse://"):
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
141
|
+
parsed_uri = urlparse(uri)
|
|
142
|
+
|
|
143
|
+
username = parsed_uri.username
|
|
144
|
+
if not username:
|
|
145
|
+
raise ValueError(
|
|
146
|
+
"A username is required to connect to the ClickHouse database."
|
|
147
|
+
)
|
|
148
|
+
|
|
149
|
+
password = parsed_uri.password
|
|
150
|
+
if not password:
|
|
151
|
+
raise ValueError(
|
|
152
|
+
"A password is required to authenticate with the ClickHouse database."
|
|
153
|
+
)
|
|
154
|
+
|
|
155
|
+
host = parsed_uri.hostname
|
|
156
|
+
if not host:
|
|
157
|
+
raise ValueError(
|
|
158
|
+
"The hostname or IP address of the ClickHouse server is required to establish a connection."
|
|
159
|
+
)
|
|
160
|
+
|
|
161
|
+
port = parsed_uri.port
|
|
162
|
+
if not port:
|
|
163
|
+
raise ValueError(
|
|
164
|
+
"The TCP port of the ClickHouse server is required to establish a connection."
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
query_params = parse_qs(parsed_uri.query)
|
|
168
|
+
|
|
169
|
+
if "http_port" in query_params:
|
|
170
|
+
del query_params["http_port"]
|
|
171
|
+
|
|
172
|
+
if "secure" not in query_params:
|
|
173
|
+
query_params["secure"] = ["1"]
|
|
174
|
+
|
|
175
|
+
uri = parsed_uri._replace(
|
|
176
|
+
scheme="clickhouse+native",
|
|
177
|
+
query=urlencode(query_params, doseq=True),
|
|
178
|
+
).geturl()
|
|
142
179
|
|
|
143
180
|
query_adapters = []
|
|
144
181
|
if kwargs.get("sql_limit"):
|
|
@@ -1754,7 +1791,7 @@ class AppLovinSource:
|
|
|
1754
1791
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1755
1792
|
if kwargs.get("incremental_key") is not None:
|
|
1756
1793
|
raise ValueError(
|
|
1757
|
-
"
|
|
1794
|
+
"Applovin takes care of incrementality on its own, you should not provide incremental_key"
|
|
1758
1795
|
)
|
|
1759
1796
|
|
|
1760
1797
|
parsed_uri = urlparse(uri)
|
|
@@ -1836,6 +1873,34 @@ class ApplovinMaxSource:
|
|
|
1836
1873
|
).with_resources(table)
|
|
1837
1874
|
|
|
1838
1875
|
|
|
1876
|
+
class SalesforceSource:
|
|
1877
|
+
def handles_incrementality(self) -> bool:
|
|
1878
|
+
return True
|
|
1879
|
+
|
|
1880
|
+
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1881
|
+
if kwargs.get("incremental_key"):
|
|
1882
|
+
raise ValueError(
|
|
1883
|
+
"Salesforce takes care of incrementality on its own, you should not provide incremental_key"
|
|
1884
|
+
)
|
|
1885
|
+
|
|
1886
|
+
params = parse_qs(urlparse(uri).query)
|
|
1887
|
+
creds = {
|
|
1888
|
+
"username": params.get("username", [None])[0],
|
|
1889
|
+
"password": params.get("password", [None])[0],
|
|
1890
|
+
"token": params.get("token", [None])[0],
|
|
1891
|
+
}
|
|
1892
|
+
for k, v in creds.items():
|
|
1893
|
+
if v is None:
|
|
1894
|
+
raise MissingValueError(k, "Salesforce")
|
|
1895
|
+
|
|
1896
|
+
src = salesforce_source(**creds) # type: ignore
|
|
1897
|
+
|
|
1898
|
+
if table not in src.resources:
|
|
1899
|
+
raise UnsupportedResourceError(table, "Salesforce")
|
|
1900
|
+
|
|
1901
|
+
return src.with_resources(table)
|
|
1902
|
+
|
|
1903
|
+
|
|
1839
1904
|
class PersonioSource:
|
|
1840
1905
|
def handles_incrementality(self) -> bool:
|
|
1841
1906
|
return True
|
|
@@ -1874,7 +1939,7 @@ class PersonioSource:
|
|
|
1874
1939
|
"custom_reports_list",
|
|
1875
1940
|
]:
|
|
1876
1941
|
raise UnsupportedResourceError(table, "Personio")
|
|
1877
|
-
|
|
1942
|
+
|
|
1878
1943
|
return personio_source(
|
|
1879
1944
|
client_id=client_id[0],
|
|
1880
1945
|
client_secret=client_secret[0],
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.11
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -18,42 +18,43 @@ Requires-Dist: asana==3.2.3
|
|
|
18
18
|
Requires-Dist: clickhouse-connect==0.8.14
|
|
19
19
|
Requires-Dist: clickhouse-driver==0.2.9
|
|
20
20
|
Requires-Dist: clickhouse-sqlalchemy==0.2.7
|
|
21
|
-
Requires-Dist: confluent-kafka>=2.
|
|
21
|
+
Requires-Dist: confluent-kafka>=2.8.0
|
|
22
22
|
Requires-Dist: databricks-sql-connector==2.9.3
|
|
23
23
|
Requires-Dist: dataclasses-json==0.6.7
|
|
24
|
-
Requires-Dist: dlt==1.
|
|
25
|
-
Requires-Dist: duckdb-engine==0.
|
|
26
|
-
Requires-Dist: duckdb==1.
|
|
24
|
+
Requires-Dist: dlt==1.6.1
|
|
25
|
+
Requires-Dist: duckdb-engine==0.15.0
|
|
26
|
+
Requires-Dist: duckdb==1.2.0
|
|
27
27
|
Requires-Dist: facebook-business==20.0.0
|
|
28
28
|
Requires-Dist: flatten-json==0.1.14
|
|
29
29
|
Requires-Dist: gcsfs==2024.10.0
|
|
30
30
|
Requires-Dist: google-ads==25.1.0
|
|
31
|
-
Requires-Dist: google-analytics-data==0.18.
|
|
31
|
+
Requires-Dist: google-analytics-data==0.18.17
|
|
32
32
|
Requires-Dist: google-api-python-client==2.130.0
|
|
33
33
|
Requires-Dist: google-cloud-bigquery-storage==2.24.0
|
|
34
|
-
Requires-Dist: mysql-connector-python==9.
|
|
34
|
+
Requires-Dist: mysql-connector-python==9.2.0
|
|
35
35
|
Requires-Dist: pendulum==3.0.0
|
|
36
36
|
Requires-Dist: psutil==6.1.1
|
|
37
37
|
Requires-Dist: psycopg2-binary==2.9.10
|
|
38
38
|
Requires-Dist: py-machineid==0.6.0
|
|
39
39
|
Requires-Dist: pyairtable==2.3.3
|
|
40
40
|
Requires-Dist: pyarrow==18.1.0
|
|
41
|
-
Requires-Dist: pyathena==3.
|
|
42
|
-
Requires-Dist: pymongo==4.
|
|
41
|
+
Requires-Dist: pyathena==3.12.2
|
|
42
|
+
Requires-Dist: pymongo==4.11.1
|
|
43
43
|
Requires-Dist: pymysql==1.1.1
|
|
44
44
|
Requires-Dist: pyrate-limiter==3.7.0
|
|
45
45
|
Requires-Dist: redshift-connector==2.1.5
|
|
46
46
|
Requires-Dist: rich==13.9.4
|
|
47
47
|
Requires-Dist: rudder-sdk-python==2.1.4
|
|
48
48
|
Requires-Dist: s3fs==2024.10.0
|
|
49
|
+
Requires-Dist: simple-salesforce==1.12.6
|
|
49
50
|
Requires-Dist: snowflake-sqlalchemy==1.6.1
|
|
50
|
-
Requires-Dist: sqlalchemy-bigquery==1.12.
|
|
51
|
+
Requires-Dist: sqlalchemy-bigquery==1.12.1
|
|
51
52
|
Requires-Dist: sqlalchemy-hana==2.0.0
|
|
52
53
|
Requires-Dist: sqlalchemy-redshift==0.8.14
|
|
53
54
|
Requires-Dist: sqlalchemy2-stubs==0.0.2a38
|
|
54
55
|
Requires-Dist: sqlalchemy==1.4.52
|
|
55
56
|
Requires-Dist: stripe==10.7.0
|
|
56
|
-
Requires-Dist: tqdm==4.67.
|
|
57
|
+
Requires-Dist: tqdm==4.67.1
|
|
57
58
|
Requires-Dist: typer==0.13.1
|
|
58
59
|
Requires-Dist: types-requests==2.32.0.20240907
|
|
59
60
|
Provides-Extra: odbc
|
|
@@ -312,6 +313,11 @@ Pull requests are welcome. However, please open an issue first to discuss what y
|
|
|
312
313
|
<td>✅</td>
|
|
313
314
|
<td>-</td>
|
|
314
315
|
</tr>
|
|
316
|
+
<tr>
|
|
317
|
+
<td>Salesforce</td>
|
|
318
|
+
<td>✅</td>
|
|
319
|
+
<td>-</td>
|
|
320
|
+
</tr>
|
|
315
321
|
<tr>
|
|
316
322
|
<td>Shopify</td>
|
|
317
323
|
<td>✅</td>
|
|
@@ -1,20 +1,20 @@
|
|
|
1
1
|
ingestr/main.py,sha256=ufn8AcM2ID80ChUApJzYDjnQaurMXOkYfTm6GzAggSQ,24746
|
|
2
2
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
3
3
|
ingestr/src/blob.py,sha256=LtEZWoUhm5i2aKerdgEpLtNCf3fdhGGMM4td-LRZVbY,1407
|
|
4
|
-
ingestr/src/buildinfo.py,sha256=
|
|
4
|
+
ingestr/src/buildinfo.py,sha256=PnFKBMVizeXpYaYJ6rkY9m_oU0QCJzbLAOJyEQ8gyRg,21
|
|
5
5
|
ingestr/src/destinations.py,sha256=vrGij4qMPCdXTMIimROWBJFqzOqCM4DFmgyubgSHejA,11279
|
|
6
6
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
7
|
-
ingestr/src/factory.py,sha256=
|
|
7
|
+
ingestr/src/factory.py,sha256=dOdY4fzeQ-2dgFBGIDFD5ilxpYNfCVqQOureuWzOL-w,5127
|
|
8
8
|
ingestr/src/filters.py,sha256=0JQXeAr2APFMnW2sd-6BlAMWv93bXV17j8b5MM8sHmM,580
|
|
9
9
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
10
|
-
ingestr/src/sources.py,sha256=
|
|
10
|
+
ingestr/src/sources.py,sha256=YlWokgTZoeMQ6PVb9UVU3I99R0cdhkYjEzPf5LNGs30,68582
|
|
11
11
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
12
12
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
13
13
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
14
14
|
ingestr/src/adjust/__init__.py,sha256=ULjtJqrNS6XDvUyGl0tjl12-tLyXlCgeFe2icTbtu3Q,3255
|
|
15
15
|
ingestr/src/adjust/adjust_helpers.py,sha256=av97NPSn-hQtTbAC0vUSCAWYePmOiG5R-DGdMssm7FQ,3646
|
|
16
16
|
ingestr/src/airtable/__init__.py,sha256=GHWYrjI2qhs_JihdNJysB0Ni3bzqT_MLXn_S9_Q5zRA,2775
|
|
17
|
-
ingestr/src/applovin/__init__.py,sha256=
|
|
17
|
+
ingestr/src/applovin/__init__.py,sha256=X_YCLppPrnL8KXfYWICE_uDfMzHHH3JZ-DBGZ1RlaOI,6984
|
|
18
18
|
ingestr/src/applovin_max/__init__.py,sha256=1NUOeJzRyZZQ95KEirbrlSrk-8SNc9JrlM_5pGgBgHg,2878
|
|
19
19
|
ingestr/src/appsflyer/_init_.py,sha256=ne2-9FQ654Drtd3GkKQv8Bwb6LEqCnJw49MfO5Jyzgs,739
|
|
20
20
|
ingestr/src/appsflyer/client.py,sha256=TNmwakLzmO6DZW3wcfLfQRl7aNBHgFqSsk4ef-MmJ1w,3084
|
|
@@ -76,6 +76,8 @@ ingestr/src/notion/helpers/client.py,sha256=QXuudkf5Zzff98HRsCqA1g1EZWIrnfn1falP
|
|
|
76
76
|
ingestr/src/notion/helpers/database.py,sha256=gigPibTeVefP3lA-8w4aOwX67pj7RlciPk5koDs1ry8,2737
|
|
77
77
|
ingestr/src/personio/__init__.py,sha256=CQ8XX8Q8BG-wgoen3emhe_r8Cx414Fux7P8jQNawWvY,11646
|
|
78
78
|
ingestr/src/personio/helpers.py,sha256=OmeMzfg4MVtpI7f75D3-9OGZb8SDsKyz0svNm1zJLTw,2900
|
|
79
|
+
ingestr/src/salesforce/__init__.py,sha256=2hik5pRrxVODdDTlUEMoyccNC07zozjnxkMHcjMT1qA,4558
|
|
80
|
+
ingestr/src/salesforce/helpers.py,sha256=QTdazBt-qRTBbCQMZnyclIaDQFmBixBy_RDKD00Lt-8,2492
|
|
79
81
|
ingestr/src/shopify/__init__.py,sha256=PF_6VQnS065Br1UzSIekTVXBu3WtrMQL_v5CfbfaX5Y,63151
|
|
80
82
|
ingestr/src/shopify/exceptions.py,sha256=BhV3lIVWeBt8Eh4CWGW_REFJpGCzvW6-62yZrBWa3nQ,50
|
|
81
83
|
ingestr/src/shopify/helpers.py,sha256=NfHD6lWXe88ybR0ri-FCQuh2Vf8l5WG0a0FVjmdoSC4,6296
|
|
@@ -106,8 +108,8 @@ ingestr/testdata/delete_insert_part2.csv,sha256=B_KUzpzbNdDY_n7wWop1mT2cz36TmayS
|
|
|
106
108
|
ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ7ZqYN0,276
|
|
107
109
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
108
110
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
109
|
-
ingestr-0.13.
|
|
110
|
-
ingestr-0.13.
|
|
111
|
-
ingestr-0.13.
|
|
112
|
-
ingestr-0.13.
|
|
113
|
-
ingestr-0.13.
|
|
111
|
+
ingestr-0.13.11.dist-info/METADATA,sha256=8vjvshEDHgAZEMt3ykbUSlEl_Ky0KtHf6p6vjT6RDGI,9171
|
|
112
|
+
ingestr-0.13.11.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
113
|
+
ingestr-0.13.11.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
114
|
+
ingestr-0.13.11.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
115
|
+
ingestr-0.13.11.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|