ingestr 0.13.79__py3-none-any.whl → 0.13.81__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of ingestr might be problematic. Click here for more details.
- ingestr/src/buildinfo.py +1 -1
- ingestr/src/facebook_ads/__init__.py +15 -16
- ingestr/src/facebook_ads/helpers.py +47 -1
- ingestr/src/linear/__init__.py +62 -42
- ingestr/src/linear/helpers.py +19 -38
- ingestr/src/sources.py +110 -20
- {ingestr-0.13.79.dist-info → ingestr-0.13.81.dist-info}/METADATA +1 -1
- {ingestr-0.13.79.dist-info → ingestr-0.13.81.dist-info}/RECORD +11 -11
- {ingestr-0.13.79.dist-info → ingestr-0.13.81.dist-info}/WHEEL +0 -0
- {ingestr-0.13.79.dist-info → ingestr-0.13.81.dist-info}/entry_points.txt +0 -0
- {ingestr-0.13.79.dist-info → ingestr-0.13.81.dist-info}/licenses/LICENSE.md +0 -0
ingestr/src/buildinfo.py
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
version = "v0.13.
|
|
1
|
+
version = "v0.13.81"
|
|
@@ -22,12 +22,8 @@ from .settings import (
|
|
|
22
22
|
DEFAULT_ADCREATIVE_FIELDS,
|
|
23
23
|
DEFAULT_ADSET_FIELDS,
|
|
24
24
|
DEFAULT_CAMPAIGN_FIELDS,
|
|
25
|
-
DEFAULT_INSIGHT_FIELDS,
|
|
26
25
|
DEFAULT_LEAD_FIELDS,
|
|
27
26
|
INSIGHT_FIELDS_TYPES,
|
|
28
|
-
INSIGHTS_BREAKDOWNS_OPTIONS,
|
|
29
|
-
INVALID_INSIGHTS_FIELDS,
|
|
30
|
-
TInsightsBreakdownOptions,
|
|
31
27
|
TInsightsLevels,
|
|
32
28
|
)
|
|
33
29
|
|
|
@@ -105,10 +101,9 @@ def facebook_insights_source(
|
|
|
105
101
|
account_id: str = dlt.config.value,
|
|
106
102
|
access_token: str = dlt.secrets.value,
|
|
107
103
|
initial_load_past_days: int = 1,
|
|
108
|
-
|
|
109
|
-
|
|
104
|
+
dimensions: Sequence[str] = None,
|
|
105
|
+
fields: Sequence[str] = None,
|
|
110
106
|
time_increment_days: int = 1,
|
|
111
|
-
breakdowns: TInsightsBreakdownOptions = "ads_insights",
|
|
112
107
|
action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
|
|
113
108
|
level: TInsightsLevels = "ad",
|
|
114
109
|
action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
|
|
@@ -117,6 +112,9 @@ def facebook_insights_source(
|
|
|
117
112
|
app_api_version: str = None,
|
|
118
113
|
start_date: pendulum.DateTime | None = None,
|
|
119
114
|
end_date: pendulum.DateTime | None = None,
|
|
115
|
+
insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
|
|
116
|
+
insights_max_wait_to_start_seconds: int = 60 * 30,
|
|
117
|
+
insights_max_async_sleep_seconds: int = 20,
|
|
120
118
|
) -> DltResource:
|
|
121
119
|
"""Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
|
|
122
120
|
|
|
@@ -152,6 +150,11 @@ def facebook_insights_source(
|
|
|
152
150
|
if start_date is None:
|
|
153
151
|
start_date = pendulum.today().subtract(days=initial_load_past_days)
|
|
154
152
|
|
|
153
|
+
if dimensions is None:
|
|
154
|
+
dimensions = []
|
|
155
|
+
if fields is None:
|
|
156
|
+
fields = []
|
|
157
|
+
|
|
155
158
|
columns = {}
|
|
156
159
|
for field in fields:
|
|
157
160
|
if field in INSIGHT_FIELDS_TYPES:
|
|
@@ -184,15 +187,9 @@ def facebook_insights_source(
|
|
|
184
187
|
query = {
|
|
185
188
|
"level": level,
|
|
186
189
|
"action_breakdowns": list(action_breakdowns),
|
|
187
|
-
"breakdowns":
|
|
188
|
-
INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
|
|
189
|
-
),
|
|
190
|
+
"breakdowns": dimensions,
|
|
190
191
|
"limit": batch_size,
|
|
191
|
-
"fields":
|
|
192
|
-
set(fields)
|
|
193
|
-
.union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
|
|
194
|
-
.difference(INVALID_INSIGHTS_FIELDS)
|
|
195
|
-
),
|
|
192
|
+
"fields": fields,
|
|
196
193
|
"time_increment": time_increment_days,
|
|
197
194
|
"action_attribution_windows": list(action_attribution_windows),
|
|
198
195
|
"time_ranges": [
|
|
@@ -206,7 +203,9 @@ def facebook_insights_source(
|
|
|
206
203
|
}
|
|
207
204
|
job = execute_job(
|
|
208
205
|
account.get_insights(params=query, is_async=True),
|
|
209
|
-
insights_max_async_sleep_seconds=
|
|
206
|
+
insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
|
|
207
|
+
insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
|
|
208
|
+
insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
|
|
210
209
|
)
|
|
211
210
|
output = list(map(process_report_item, job.get_result()))
|
|
212
211
|
yield output
|
|
@@ -144,7 +144,7 @@ def execute_job(
|
|
|
144
144
|
raise InsightsJobTimeout(
|
|
145
145
|
"facebook_insights",
|
|
146
146
|
pretty_error_message.format(
|
|
147
|
-
job_id, insights_max_wait_to_finish_seconds
|
|
147
|
+
job_id, insights_max_wait_to_finish_seconds
|
|
148
148
|
),
|
|
149
149
|
)
|
|
150
150
|
|
|
@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
|
|
|
229
229
|
logger.error(
|
|
230
230
|
f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
|
|
231
231
|
)
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
|
|
235
|
+
import typing
|
|
236
|
+
|
|
237
|
+
from ingestr.src.facebook_ads.settings import (
|
|
238
|
+
INSIGHTS_BREAKDOWNS_OPTIONS,
|
|
239
|
+
TInsightsBreakdownOptions,
|
|
240
|
+
TInsightsLevels,
|
|
241
|
+
)
|
|
242
|
+
|
|
243
|
+
parts = table.split(":")
|
|
244
|
+
|
|
245
|
+
source_kwargs = {}
|
|
246
|
+
|
|
247
|
+
breakdown_type = parts[1]
|
|
248
|
+
|
|
249
|
+
valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
|
|
250
|
+
if breakdown_type in valid_breakdowns:
|
|
251
|
+
dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
|
|
252
|
+
fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
|
|
253
|
+
source_kwargs["dimensions"] = dimensions
|
|
254
|
+
source_kwargs["fields"] = fields
|
|
255
|
+
else:
|
|
256
|
+
dimensions = breakdown_type.split(",")
|
|
257
|
+
valid_levels = list(typing.get_args(TInsightsLevels))
|
|
258
|
+
level = None
|
|
259
|
+
for valid_level in reversed(valid_levels):
|
|
260
|
+
if valid_level in dimensions:
|
|
261
|
+
level = valid_level
|
|
262
|
+
dimensions.remove(valid_level)
|
|
263
|
+
break
|
|
264
|
+
|
|
265
|
+
source_kwargs["level"] = level
|
|
266
|
+
source_kwargs["dimensions"] = dimensions
|
|
267
|
+
|
|
268
|
+
# If custom metrics are provided, parse them
|
|
269
|
+
if len(parts) == 3:
|
|
270
|
+
fields = [f.strip() for f in parts[2].split(",") if f.strip()]
|
|
271
|
+
if not fields:
|
|
272
|
+
raise ValueError(
|
|
273
|
+
"Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
|
|
274
|
+
)
|
|
275
|
+
source_kwargs["fields"] = fields
|
|
276
|
+
|
|
277
|
+
return source_kwargs
|
ingestr/src/linear/__init__.py
CHANGED
|
@@ -3,7 +3,23 @@ from typing import Any, Dict, Iterable, Iterator
|
|
|
3
3
|
import dlt
|
|
4
4
|
import pendulum
|
|
5
5
|
|
|
6
|
-
from .helpers import
|
|
6
|
+
from .helpers import _paginate, normalize_dictionaries
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def _get_date_range(updated_at, start_date):
|
|
10
|
+
"""Extract current start and end dates from incremental state."""
|
|
11
|
+
if updated_at.last_value:
|
|
12
|
+
current_start_date = pendulum.parse(updated_at.last_value)
|
|
13
|
+
else:
|
|
14
|
+
current_start_date = pendulum.parse(start_date)
|
|
15
|
+
|
|
16
|
+
if updated_at.end_value:
|
|
17
|
+
current_end_date = pendulum.parse(updated_at.end_value)
|
|
18
|
+
else:
|
|
19
|
+
current_end_date = pendulum.now(tz="UTC")
|
|
20
|
+
|
|
21
|
+
return current_start_date, current_end_date
|
|
22
|
+
|
|
7
23
|
|
|
8
24
|
ISSUES_QUERY = """
|
|
9
25
|
query Issues($cursor: String) {
|
|
@@ -84,6 +100,25 @@ query Users($cursor: String) {
|
|
|
84
100
|
}
|
|
85
101
|
}
|
|
86
102
|
"""
|
|
103
|
+
WORKFLOW_STATES_QUERY = """
|
|
104
|
+
query WorkflowStates($cursor: String) {
|
|
105
|
+
workflowStates(first: 50, after: $cursor) {
|
|
106
|
+
nodes {
|
|
107
|
+
archivedAt
|
|
108
|
+
color
|
|
109
|
+
createdAt
|
|
110
|
+
id
|
|
111
|
+
inheritedFrom { id }
|
|
112
|
+
name
|
|
113
|
+
position
|
|
114
|
+
team { id }
|
|
115
|
+
type
|
|
116
|
+
updatedAt
|
|
117
|
+
}
|
|
118
|
+
pageInfo { hasNextPage endCursor }
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
"""
|
|
87
122
|
|
|
88
123
|
|
|
89
124
|
@dlt.source(name="linear", max_table_nesting=0)
|
|
@@ -102,20 +137,12 @@ def linear_source(
|
|
|
102
137
|
range_end="closed",
|
|
103
138
|
),
|
|
104
139
|
) -> Iterator[Dict[str, Any]]:
|
|
105
|
-
|
|
106
|
-
current_start_date = pendulum.parse(updated_at.last_value)
|
|
107
|
-
else:
|
|
108
|
-
current_start_date = pendulum.parse(start_date)
|
|
109
|
-
|
|
110
|
-
if updated_at.end_value:
|
|
111
|
-
current_end_date = pendulum.parse(updated_at.end_value)
|
|
112
|
-
else:
|
|
113
|
-
current_end_date = pendulum.now(tz="UTC")
|
|
140
|
+
current_start_date, current_end_date = _get_date_range(updated_at, start_date)
|
|
114
141
|
|
|
115
142
|
for item in _paginate(api_key, ISSUES_QUERY, "issues"):
|
|
116
143
|
if pendulum.parse(item["updatedAt"]) >= current_start_date:
|
|
117
144
|
if pendulum.parse(item["updatedAt"]) <= current_end_date:
|
|
118
|
-
yield
|
|
145
|
+
yield normalize_dictionaries(item)
|
|
119
146
|
|
|
120
147
|
@dlt.resource(name="projects", primary_key="id", write_disposition="merge")
|
|
121
148
|
def projects(
|
|
@@ -127,20 +154,12 @@ def linear_source(
|
|
|
127
154
|
range_end="closed",
|
|
128
155
|
),
|
|
129
156
|
) -> Iterator[Dict[str, Any]]:
|
|
130
|
-
|
|
131
|
-
current_start_date = pendulum.parse(updated_at.last_value)
|
|
132
|
-
else:
|
|
133
|
-
current_start_date = pendulum.parse(start_date)
|
|
134
|
-
|
|
135
|
-
if updated_at.end_value:
|
|
136
|
-
current_end_date = pendulum.parse(updated_at.end_value)
|
|
137
|
-
else:
|
|
138
|
-
current_end_date = pendulum.now(tz="UTC")
|
|
157
|
+
current_start_date, current_end_date = _get_date_range(updated_at, start_date)
|
|
139
158
|
|
|
140
159
|
for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
|
|
141
160
|
if pendulum.parse(item["updatedAt"]) >= current_start_date:
|
|
142
161
|
if pendulum.parse(item["updatedAt"]) <= current_end_date:
|
|
143
|
-
yield item
|
|
162
|
+
yield normalize_dictionaries(item)
|
|
144
163
|
|
|
145
164
|
@dlt.resource(name="teams", primary_key="id", write_disposition="merge")
|
|
146
165
|
def teams(
|
|
@@ -153,21 +172,13 @@ def linear_source(
|
|
|
153
172
|
),
|
|
154
173
|
) -> Iterator[Dict[str, Any]]:
|
|
155
174
|
print(start_date)
|
|
156
|
-
|
|
157
|
-
current_start_date = pendulum.parse(updated_at.last_value)
|
|
158
|
-
else:
|
|
159
|
-
current_start_date = pendulum.parse(start_date)
|
|
175
|
+
current_start_date, current_end_date = _get_date_range(updated_at, start_date)
|
|
160
176
|
print(current_start_date)
|
|
161
177
|
|
|
162
|
-
if updated_at.end_value:
|
|
163
|
-
current_end_date = pendulum.parse(updated_at.end_value)
|
|
164
|
-
else:
|
|
165
|
-
current_end_date = pendulum.now(tz="UTC")
|
|
166
|
-
|
|
167
178
|
for item in _paginate(api_key, TEAMS_QUERY, "teams"):
|
|
168
179
|
if pendulum.parse(item["updatedAt"]) >= current_start_date:
|
|
169
180
|
if pendulum.parse(item["updatedAt"]) <= current_end_date:
|
|
170
|
-
yield
|
|
181
|
+
yield normalize_dictionaries(item)
|
|
171
182
|
|
|
172
183
|
@dlt.resource(name="users", primary_key="id", write_disposition="merge")
|
|
173
184
|
def users(
|
|
@@ -179,19 +190,28 @@ def linear_source(
|
|
|
179
190
|
range_end="closed",
|
|
180
191
|
),
|
|
181
192
|
) -> Iterator[Dict[str, Any]]:
|
|
182
|
-
|
|
183
|
-
current_start_date = pendulum.parse(updated_at.last_value)
|
|
184
|
-
else:
|
|
185
|
-
current_start_date = pendulum.parse(start_date)
|
|
186
|
-
|
|
187
|
-
if updated_at.end_value:
|
|
188
|
-
current_end_date = pendulum.parse(updated_at.end_value)
|
|
189
|
-
else:
|
|
190
|
-
current_end_date = pendulum.now(tz="UTC")
|
|
193
|
+
current_start_date, current_end_date = _get_date_range(updated_at, start_date)
|
|
191
194
|
|
|
192
195
|
for item in _paginate(api_key, USERS_QUERY, "users"):
|
|
193
196
|
if pendulum.parse(item["updatedAt"]) >= current_start_date:
|
|
194
197
|
if pendulum.parse(item["updatedAt"]) <= current_end_date:
|
|
195
|
-
yield item
|
|
198
|
+
yield normalize_dictionaries(item)
|
|
199
|
+
|
|
200
|
+
@dlt.resource(name="workflow_states", primary_key="id", write_disposition="merge")
|
|
201
|
+
def workflow_states(
|
|
202
|
+
updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
|
|
203
|
+
"updatedAt",
|
|
204
|
+
initial_value=start_date.isoformat(),
|
|
205
|
+
end_value=end_date.isoformat() if end_date else None,
|
|
206
|
+
range_start="closed",
|
|
207
|
+
range_end="closed",
|
|
208
|
+
),
|
|
209
|
+
) -> Iterator[Dict[str, Any]]:
|
|
210
|
+
current_start_date, current_end_date = _get_date_range(updated_at, start_date)
|
|
211
|
+
|
|
212
|
+
for item in _paginate(api_key, WORKFLOW_STATES_QUERY, "workflowStates"):
|
|
213
|
+
if pendulum.parse(item["updatedAt"]) >= current_start_date:
|
|
214
|
+
if pendulum.parse(item["updatedAt"]) <= current_end_date:
|
|
215
|
+
yield normalize_dictionaries(item)
|
|
196
216
|
|
|
197
|
-
return issues, projects, teams, users
|
|
217
|
+
return [issues, projects, teams, users, workflow_states]
|
ingestr/src/linear/helpers.py
CHANGED
|
@@ -32,41 +32,22 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
|
|
|
32
32
|
cursor = data["pageInfo"]["endCursor"]
|
|
33
33
|
|
|
34
34
|
|
|
35
|
-
def
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
for key, value in
|
|
44
|
-
if
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
"labels",
|
|
55
|
-
"subtasks",
|
|
56
|
-
"projects",
|
|
57
|
-
"memberships",
|
|
58
|
-
"members",
|
|
59
|
-
]
|
|
60
|
-
for field in json_fields:
|
|
61
|
-
if item.get(field):
|
|
62
|
-
item[f"{field}"] = item[field].get("nodes", [])
|
|
63
|
-
|
|
64
|
-
return item
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
|
|
68
|
-
json_fields = ["memberships", "members", "projects"]
|
|
69
|
-
for field in json_fields:
|
|
70
|
-
if item.get(field):
|
|
71
|
-
item[f"{field}"] = item[field].get("nodes", [])
|
|
72
|
-
return item
|
|
35
|
+
def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
|
|
36
|
+
"""
|
|
37
|
+
Automatically normalize dictionary fields by detecting their structure:
|
|
38
|
+
- Convert nested objects with 'id' field to {field_name}_id
|
|
39
|
+
- Convert objects with 'nodes' field to arrays
|
|
40
|
+
"""
|
|
41
|
+
normalized_item = item.copy()
|
|
42
|
+
|
|
43
|
+
for key, value in list(normalized_item.items()):
|
|
44
|
+
if isinstance(value, dict):
|
|
45
|
+
# If the dict has an 'id' field, replace with {key}_id
|
|
46
|
+
if "id" in value:
|
|
47
|
+
normalized_item[f"{key}_id"] = value["id"]
|
|
48
|
+
del normalized_item[key]
|
|
49
|
+
# If the dict has 'nodes' field, extract the nodes array
|
|
50
|
+
elif "nodes" in value:
|
|
51
|
+
normalized_item[key] = value["nodes"]
|
|
52
|
+
|
|
53
|
+
return normalized_item
|
ingestr/src/sources.py
CHANGED
|
@@ -702,6 +702,11 @@ class ShopifySource:
|
|
|
702
702
|
return True
|
|
703
703
|
|
|
704
704
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
705
|
+
if kwargs.get("incremental_key"):
|
|
706
|
+
raise ValueError(
|
|
707
|
+
"Shopify takes care of incrementality on its own, you should not provide incremental_key"
|
|
708
|
+
)
|
|
709
|
+
|
|
705
710
|
source_fields = urlparse(uri)
|
|
706
711
|
source_params = parse_qs(source_fields.query)
|
|
707
712
|
api_key = source_params.get("api_key")
|
|
@@ -1003,6 +1008,16 @@ class FacebookAdsSource:
|
|
|
1003
1008
|
facebook_insights_source,
|
|
1004
1009
|
)
|
|
1005
1010
|
|
|
1011
|
+
insights_max_wait_to_finish_seconds = source_params.get(
|
|
1012
|
+
"insights_max_wait_to_finish_seconds", [60 * 60 * 4]
|
|
1013
|
+
)
|
|
1014
|
+
insights_max_wait_to_start_seconds = source_params.get(
|
|
1015
|
+
"insights_max_wait_to_start_seconds", [60 * 30]
|
|
1016
|
+
)
|
|
1017
|
+
insights_max_async_sleep_seconds = source_params.get(
|
|
1018
|
+
"insights_max_async_sleep_seconds", [20]
|
|
1019
|
+
)
|
|
1020
|
+
|
|
1006
1021
|
endpoint = None
|
|
1007
1022
|
if table in ["campaigns", "ad_sets", "ad_creatives", "ads", "leads"]:
|
|
1008
1023
|
endpoint = table
|
|
@@ -1012,6 +1027,13 @@ class FacebookAdsSource:
|
|
|
1012
1027
|
account_id=account_id[0],
|
|
1013
1028
|
start_date=kwargs.get("interval_start"),
|
|
1014
1029
|
end_date=kwargs.get("interval_end"),
|
|
1030
|
+
insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds[
|
|
1031
|
+
0
|
|
1032
|
+
],
|
|
1033
|
+
insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds[
|
|
1034
|
+
0
|
|
1035
|
+
],
|
|
1036
|
+
insights_max_async_sleep_seconds=insights_max_async_sleep_seconds[0],
|
|
1015
1037
|
).with_resources("facebook_insights")
|
|
1016
1038
|
elif table.startswith("facebook_insights:"):
|
|
1017
1039
|
# Parse custom breakdowns and metrics from table name
|
|
@@ -1034,33 +1056,16 @@ class FacebookAdsSource:
|
|
|
1034
1056
|
# Validate breakdown type against available options from settings
|
|
1035
1057
|
import typing
|
|
1036
1058
|
|
|
1037
|
-
from ingestr.src.facebook_ads.
|
|
1038
|
-
|
|
1039
|
-
# Get valid breakdown options from the type definition
|
|
1040
|
-
valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
|
|
1041
|
-
|
|
1042
|
-
if breakdown_type not in valid_breakdowns:
|
|
1043
|
-
raise ValueError(
|
|
1044
|
-
f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
|
|
1045
|
-
)
|
|
1059
|
+
from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
|
|
1046
1060
|
|
|
1047
1061
|
source_kwargs = {
|
|
1048
1062
|
"access_token": access_token[0],
|
|
1049
1063
|
"account_id": account_id[0],
|
|
1050
1064
|
"start_date": kwargs.get("interval_start"),
|
|
1051
1065
|
"end_date": kwargs.get("interval_end"),
|
|
1052
|
-
"breakdowns": breakdown_type,
|
|
1053
1066
|
}
|
|
1054
1067
|
|
|
1055
|
-
|
|
1056
|
-
if len(parts) == 3:
|
|
1057
|
-
fields = [f.strip() for f in parts[2].split(",") if f.strip()]
|
|
1058
|
-
if not fields:
|
|
1059
|
-
raise ValueError(
|
|
1060
|
-
"Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
|
|
1061
|
-
)
|
|
1062
|
-
source_kwargs["fields"] = fields
|
|
1063
|
-
|
|
1068
|
+
source_kwargs.update(parse_insights_table_to_source_kwargs(table))
|
|
1064
1069
|
return facebook_insights_source(**source_kwargs).with_resources(
|
|
1065
1070
|
"facebook_insights"
|
|
1066
1071
|
)
|
|
@@ -1652,6 +1657,11 @@ class TikTokSource:
|
|
|
1652
1657
|
return True
|
|
1653
1658
|
|
|
1654
1659
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1660
|
+
if kwargs.get("incremental_key"):
|
|
1661
|
+
raise ValueError(
|
|
1662
|
+
"TikTok takes care of incrementality on its own, you should not provide incremental_key"
|
|
1663
|
+
)
|
|
1664
|
+
|
|
1655
1665
|
endpoint = "custom_reports"
|
|
1656
1666
|
|
|
1657
1667
|
parsed_uri = urlparse(uri)
|
|
@@ -1874,6 +1884,11 @@ class GoogleAnalyticsSource:
|
|
|
1874
1884
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
1875
1885
|
import ingestr.src.google_analytics.helpers as helpers
|
|
1876
1886
|
|
|
1887
|
+
if kwargs.get("incremental_key"):
|
|
1888
|
+
raise ValueError(
|
|
1889
|
+
"Google Analytics takes care of incrementality on its own, you should not provide incremental_key"
|
|
1890
|
+
)
|
|
1891
|
+
|
|
1877
1892
|
result = helpers.parse_google_analytics_uri(uri)
|
|
1878
1893
|
credentials = result["credentials"]
|
|
1879
1894
|
property_id = result["property_id"]
|
|
@@ -2246,6 +2261,11 @@ class LinkedInAdsSource:
|
|
|
2246
2261
|
return True
|
|
2247
2262
|
|
|
2248
2263
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2264
|
+
if kwargs.get("incremental_key"):
|
|
2265
|
+
raise ValueError(
|
|
2266
|
+
"LinkedIn Ads takes care of incrementality on its own, you should not provide incremental_key"
|
|
2267
|
+
)
|
|
2268
|
+
|
|
2249
2269
|
parsed_uri = urlparse(uri)
|
|
2250
2270
|
source_fields = parse_qs(parsed_uri.query)
|
|
2251
2271
|
|
|
@@ -2329,6 +2349,11 @@ class ClickupSource:
|
|
|
2329
2349
|
return True
|
|
2330
2350
|
|
|
2331
2351
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2352
|
+
if kwargs.get("incremental_key"):
|
|
2353
|
+
raise ValueError(
|
|
2354
|
+
"ClickUp takes care of incrementality on its own, you should not provide incremental_key"
|
|
2355
|
+
)
|
|
2356
|
+
|
|
2332
2357
|
parsed_uri = urlparse(uri)
|
|
2333
2358
|
params = parse_qs(parsed_uri.query)
|
|
2334
2359
|
api_token = params.get("api_token")
|
|
@@ -2413,6 +2438,11 @@ class ApplovinMaxSource:
|
|
|
2413
2438
|
return True
|
|
2414
2439
|
|
|
2415
2440
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2441
|
+
if kwargs.get("incremental_key"):
|
|
2442
|
+
raise ValueError(
|
|
2443
|
+
"AppLovin Max takes care of incrementality on its own, you should not provide incremental_key"
|
|
2444
|
+
)
|
|
2445
|
+
|
|
2416
2446
|
parsed_uri = urlparse(uri)
|
|
2417
2447
|
params = parse_qs(parsed_uri.query)
|
|
2418
2448
|
|
|
@@ -2505,6 +2535,11 @@ class PersonioSource:
|
|
|
2505
2535
|
|
|
2506
2536
|
# applovin://?client_id=123&client_secret=123
|
|
2507
2537
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2538
|
+
if kwargs.get("incremental_key"):
|
|
2539
|
+
raise ValueError(
|
|
2540
|
+
"Personio takes care of incrementality on its own, you should not provide incremental_key"
|
|
2541
|
+
)
|
|
2542
|
+
|
|
2508
2543
|
parsed_uri = urlparse(uri)
|
|
2509
2544
|
params = parse_qs(parsed_uri.query)
|
|
2510
2545
|
|
|
@@ -2595,6 +2630,11 @@ class PipedriveSource:
|
|
|
2595
2630
|
return True
|
|
2596
2631
|
|
|
2597
2632
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2633
|
+
if kwargs.get("incremental_key"):
|
|
2634
|
+
raise ValueError(
|
|
2635
|
+
"Pipedrive takes care of incrementality on its own, you should not provide incremental_key"
|
|
2636
|
+
)
|
|
2637
|
+
|
|
2598
2638
|
parsed_uri = urlparse(uri)
|
|
2599
2639
|
params = parse_qs(parsed_uri.query)
|
|
2600
2640
|
api_key = params.get("api_token")
|
|
@@ -2677,6 +2717,11 @@ class FreshdeskSource:
|
|
|
2677
2717
|
return True
|
|
2678
2718
|
|
|
2679
2719
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2720
|
+
if kwargs.get("incremental_key"):
|
|
2721
|
+
raise ValueError(
|
|
2722
|
+
"Freshdesk takes care of incrementality on its own, you should not provide incremental_key"
|
|
2723
|
+
)
|
|
2724
|
+
|
|
2680
2725
|
parsed_uri = urlparse(uri)
|
|
2681
2726
|
domain = parsed_uri.netloc
|
|
2682
2727
|
query = parsed_uri.query
|
|
@@ -2730,6 +2775,11 @@ class TrustpilotSource:
|
|
|
2730
2775
|
return True
|
|
2731
2776
|
|
|
2732
2777
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2778
|
+
if kwargs.get("incremental_key"):
|
|
2779
|
+
raise ValueError(
|
|
2780
|
+
"Trustpilot takes care of incrementality on its own, you should not provide incremental_key"
|
|
2781
|
+
)
|
|
2782
|
+
|
|
2733
2783
|
parsed_uri = urlparse(uri)
|
|
2734
2784
|
business_unit_id = parsed_uri.netloc
|
|
2735
2785
|
params = parse_qs(parsed_uri.query)
|
|
@@ -2770,6 +2820,11 @@ class PhantombusterSource:
|
|
|
2770
2820
|
return True
|
|
2771
2821
|
|
|
2772
2822
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2823
|
+
if kwargs.get("incremental_key"):
|
|
2824
|
+
raise ValueError(
|
|
2825
|
+
"Phantombuster takes care of incrementality on its own, you should not provide incremental_key"
|
|
2826
|
+
)
|
|
2827
|
+
|
|
2773
2828
|
# phantombuster://?api_key=<api_key>
|
|
2774
2829
|
# source table = phantom_results:agent_id
|
|
2775
2830
|
parsed_uri = urlparse(uri)
|
|
@@ -2923,6 +2978,11 @@ class SolidgateSource:
|
|
|
2923
2978
|
return True
|
|
2924
2979
|
|
|
2925
2980
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
2981
|
+
if kwargs.get("incremental_key"):
|
|
2982
|
+
raise ValueError(
|
|
2983
|
+
"Solidgate takes care of incrementality on its own, you should not provide incremental_key"
|
|
2984
|
+
)
|
|
2985
|
+
|
|
2926
2986
|
parsed_uri = urlparse(uri)
|
|
2927
2987
|
query_params = parse_qs(parsed_uri.query)
|
|
2928
2988
|
public_key = query_params.get("public_key")
|
|
@@ -3016,6 +3076,11 @@ class QuickBooksSource:
|
|
|
3016
3076
|
|
|
3017
3077
|
# quickbooks://?company_id=<company_id>&client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh>&access_token=<access_token>&environment=<env>&minor_version=<version>
|
|
3018
3078
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3079
|
+
if kwargs.get("incremental_key"):
|
|
3080
|
+
raise ValueError(
|
|
3081
|
+
"QuickBooks takes care of incrementality on its own, you should not provide incremental_key"
|
|
3082
|
+
)
|
|
3083
|
+
|
|
3019
3084
|
parsed_uri = urlparse(uri)
|
|
3020
3085
|
|
|
3021
3086
|
params = parse_qs(parsed_uri.query)
|
|
@@ -3085,6 +3150,11 @@ class IsocPulseSource:
|
|
|
3085
3150
|
return True
|
|
3086
3151
|
|
|
3087
3152
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3153
|
+
if kwargs.get("incremental_key"):
|
|
3154
|
+
raise ValueError(
|
|
3155
|
+
"Internet Society Pulse takes care of incrementality on its own, you should not provide incremental_key"
|
|
3156
|
+
)
|
|
3157
|
+
|
|
3088
3158
|
parsed_uri = urlparse(uri)
|
|
3089
3159
|
params = parse_qs(parsed_uri.query)
|
|
3090
3160
|
token = params.get("token")
|
|
@@ -3120,6 +3190,11 @@ class PinterestSource:
|
|
|
3120
3190
|
return True
|
|
3121
3191
|
|
|
3122
3192
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3193
|
+
if kwargs.get("incremental_key"):
|
|
3194
|
+
raise ValueError(
|
|
3195
|
+
"Pinterest takes care of incrementality on its own, you should not provide incremental_key"
|
|
3196
|
+
)
|
|
3197
|
+
|
|
3123
3198
|
parsed = urlparse(uri)
|
|
3124
3199
|
params = parse_qs(parsed.query)
|
|
3125
3200
|
access_token = params.get("access_token")
|
|
@@ -3154,13 +3229,18 @@ class LinearSource:
|
|
|
3154
3229
|
return True
|
|
3155
3230
|
|
|
3156
3231
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3232
|
+
if kwargs.get("incremental_key"):
|
|
3233
|
+
raise ValueError(
|
|
3234
|
+
"Linear takes care of incrementality on its own, you should not provide incremental_key"
|
|
3235
|
+
)
|
|
3236
|
+
|
|
3157
3237
|
parsed_uri = urlparse(uri)
|
|
3158
3238
|
params = parse_qs(parsed_uri.query)
|
|
3159
3239
|
api_key = params.get("api_key")
|
|
3160
3240
|
if api_key is None:
|
|
3161
3241
|
raise MissingValueError("api_key", "Linear")
|
|
3162
3242
|
|
|
3163
|
-
if table not in ["issues", "projects", "teams", "users"]:
|
|
3243
|
+
if table not in ["issues", "projects", "teams", "users", "workflow_states"]:
|
|
3164
3244
|
raise UnsupportedResourceError(table, "Linear")
|
|
3165
3245
|
|
|
3166
3246
|
start_date = kwargs.get("interval_start")
|
|
@@ -3187,6 +3267,11 @@ class ZoomSource:
|
|
|
3187
3267
|
return True
|
|
3188
3268
|
|
|
3189
3269
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3270
|
+
if kwargs.get("incremental_key"):
|
|
3271
|
+
raise ValueError(
|
|
3272
|
+
"Zoom takes care of incrementality on its own, you should not provide incremental_key"
|
|
3273
|
+
)
|
|
3274
|
+
|
|
3190
3275
|
parsed = urlparse(uri)
|
|
3191
3276
|
params = parse_qs(parsed.query)
|
|
3192
3277
|
client_id = params.get("client_id")
|
|
@@ -3228,6 +3313,11 @@ class InfluxDBSource:
|
|
|
3228
3313
|
return True
|
|
3229
3314
|
|
|
3230
3315
|
def dlt_source(self, uri: str, table: str, **kwargs):
|
|
3316
|
+
if kwargs.get("incremental_key"):
|
|
3317
|
+
raise ValueError(
|
|
3318
|
+
"InfluxDB takes care of incrementality on its own, you should not provide incremental_key"
|
|
3319
|
+
)
|
|
3320
|
+
|
|
3231
3321
|
parsed_uri = urlparse(uri)
|
|
3232
3322
|
params = parse_qs(parsed_uri.query)
|
|
3233
3323
|
host = parsed_uri.hostname
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: ingestr
|
|
3
|
-
Version: 0.13.
|
|
3
|
+
Version: 0.13.81
|
|
4
4
|
Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
|
|
5
5
|
Project-URL: Homepage, https://github.com/bruin-data/ingestr
|
|
6
6
|
Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
|
|
@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
|
|
|
2
2
|
ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
|
|
3
3
|
ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
|
|
4
4
|
ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
|
|
5
|
-
ingestr/src/buildinfo.py,sha256=
|
|
5
|
+
ingestr/src/buildinfo.py,sha256=H6wsXWMjlzTHcGuLKaXl67mkAFCFJzin5gNxWfo6nMg,21
|
|
6
6
|
ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
|
|
7
7
|
ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
|
|
8
8
|
ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
|
|
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
|
|
|
11
11
|
ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
|
|
12
12
|
ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
|
|
13
13
|
ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
|
|
14
|
-
ingestr/src/sources.py,sha256=
|
|
14
|
+
ingestr/src/sources.py,sha256=wdNRqVzuR8EK8s8SzL3lZ8UE6UrqLMq-FJGkc23rrpM,119190
|
|
15
15
|
ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
|
|
16
16
|
ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
|
|
17
17
|
ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
|
|
@@ -41,9 +41,9 @@ ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBC
|
|
|
41
41
|
ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
|
|
42
42
|
ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
|
|
43
43
|
ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
|
|
44
|
-
ingestr/src/facebook_ads/__init__.py,sha256=
|
|
44
|
+
ingestr/src/facebook_ads/__init__.py,sha256=15GiovITANe0al5MI6WWLdl3LDmdBd1YpkUWBV3g6bk,9715
|
|
45
45
|
ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
|
|
46
|
-
ingestr/src/facebook_ads/helpers.py,sha256=
|
|
46
|
+
ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
|
|
47
47
|
ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
|
|
48
48
|
ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
|
|
49
49
|
ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
|
|
@@ -85,8 +85,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
|
|
|
85
85
|
ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
|
|
86
86
|
ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
|
|
87
87
|
ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
|
|
88
|
-
ingestr/src/linear/__init__.py,sha256=
|
|
89
|
-
ingestr/src/linear/helpers.py,sha256=
|
|
88
|
+
ingestr/src/linear/__init__.py,sha256=CZnRQfP44L1fy4ppnukuTr1wWKSNlPpaLFE3Xw5vJr0,6605
|
|
89
|
+
ingestr/src/linear/helpers.py,sha256=jpG9l3JhJVXk2K77iny8BMqkYHGycwS4Kmcdq2Xs7aE,1786
|
|
90
90
|
ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
|
|
91
91
|
ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
|
|
92
92
|
ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
|
|
@@ -151,8 +151,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
|
|
|
151
151
|
ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
|
|
152
152
|
ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
|
|
153
153
|
ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
|
|
154
|
-
ingestr-0.13.
|
|
155
|
-
ingestr-0.13.
|
|
156
|
-
ingestr-0.13.
|
|
157
|
-
ingestr-0.13.
|
|
158
|
-
ingestr-0.13.
|
|
154
|
+
ingestr-0.13.81.dist-info/METADATA,sha256=jjolsfHDoVMclLreJMhKFu6s1_FZ5d_eN2F8CFgEDFA,15182
|
|
155
|
+
ingestr-0.13.81.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
|
|
156
|
+
ingestr-0.13.81.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
|
|
157
|
+
ingestr-0.13.81.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
|
|
158
|
+
ingestr-0.13.81.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|