ingestr 0.13.79__py3-none-any.whl → 0.13.81__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Potentially problematic release.


This version of ingestr might be problematic. Click here for more details.

ingestr/src/buildinfo.py CHANGED
@@ -1 +1 @@
1
- version = "v0.13.79"
1
+ version = "v0.13.81"
@@ -22,12 +22,8 @@ from .settings import (
22
22
  DEFAULT_ADCREATIVE_FIELDS,
23
23
  DEFAULT_ADSET_FIELDS,
24
24
  DEFAULT_CAMPAIGN_FIELDS,
25
- DEFAULT_INSIGHT_FIELDS,
26
25
  DEFAULT_LEAD_FIELDS,
27
26
  INSIGHT_FIELDS_TYPES,
28
- INSIGHTS_BREAKDOWNS_OPTIONS,
29
- INVALID_INSIGHTS_FIELDS,
30
- TInsightsBreakdownOptions,
31
27
  TInsightsLevels,
32
28
  )
33
29
 
@@ -105,10 +101,9 @@ def facebook_insights_source(
105
101
  account_id: str = dlt.config.value,
106
102
  access_token: str = dlt.secrets.value,
107
103
  initial_load_past_days: int = 1,
108
- fields: Sequence[str] = DEFAULT_INSIGHT_FIELDS,
109
- attribution_window_days_lag: int = 7,
104
+ dimensions: Sequence[str] = None,
105
+ fields: Sequence[str] = None,
110
106
  time_increment_days: int = 1,
111
- breakdowns: TInsightsBreakdownOptions = "ads_insights",
112
107
  action_breakdowns: Sequence[str] = ALL_ACTION_BREAKDOWNS,
113
108
  level: TInsightsLevels = "ad",
114
109
  action_attribution_windows: Sequence[str] = ALL_ACTION_ATTRIBUTION_WINDOWS,
@@ -117,6 +112,9 @@ def facebook_insights_source(
117
112
  app_api_version: str = None,
118
113
  start_date: pendulum.DateTime | None = None,
119
114
  end_date: pendulum.DateTime | None = None,
115
+ insights_max_wait_to_finish_seconds: int = 60 * 60 * 4,
116
+ insights_max_wait_to_start_seconds: int = 60 * 30,
117
+ insights_max_async_sleep_seconds: int = 20,
120
118
  ) -> DltResource:
121
119
  """Incrementally loads insight reports with defined granularity level, fields, breakdowns etc.
122
120
 
@@ -152,6 +150,11 @@ def facebook_insights_source(
152
150
  if start_date is None:
153
151
  start_date = pendulum.today().subtract(days=initial_load_past_days)
154
152
 
153
+ if dimensions is None:
154
+ dimensions = []
155
+ if fields is None:
156
+ fields = []
157
+
155
158
  columns = {}
156
159
  for field in fields:
157
160
  if field in INSIGHT_FIELDS_TYPES:
@@ -184,15 +187,9 @@ def facebook_insights_source(
184
187
  query = {
185
188
  "level": level,
186
189
  "action_breakdowns": list(action_breakdowns),
187
- "breakdowns": list(
188
- INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["breakdowns"]
189
- ),
190
+ "breakdowns": dimensions,
190
191
  "limit": batch_size,
191
- "fields": list(
192
- set(fields)
193
- .union(INSIGHTS_BREAKDOWNS_OPTIONS[breakdowns]["fields"])
194
- .difference(INVALID_INSIGHTS_FIELDS)
195
- ),
192
+ "fields": fields,
196
193
  "time_increment": time_increment_days,
197
194
  "action_attribution_windows": list(action_attribution_windows),
198
195
  "time_ranges": [
@@ -206,7 +203,9 @@ def facebook_insights_source(
206
203
  }
207
204
  job = execute_job(
208
205
  account.get_insights(params=query, is_async=True),
209
- insights_max_async_sleep_seconds=20,
206
+ insights_max_async_sleep_seconds=insights_max_async_sleep_seconds,
207
+ insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds,
208
+ insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds,
210
209
  )
211
210
  output = list(map(process_report_item, job.get_result()))
212
211
  yield output
@@ -144,7 +144,7 @@ def execute_job(
144
144
  raise InsightsJobTimeout(
145
145
  "facebook_insights",
146
146
  pretty_error_message.format(
147
- job_id, insights_max_wait_to_finish_seconds // 60
147
+ job_id, insights_max_wait_to_finish_seconds
148
148
  ),
149
149
  )
150
150
 
@@ -229,3 +229,49 @@ def notify_on_token_expiration(access_token_expires_at: int = None) -> None:
229
229
  logger.error(
230
230
  f"Access Token expires in {humanize.precisedelta(pendulum.now() - expires_at)}. Replace the token now!"
231
231
  )
232
+
233
+
234
+ def parse_insights_table_to_source_kwargs(table: str) -> DictStrAny:
235
+ import typing
236
+
237
+ from ingestr.src.facebook_ads.settings import (
238
+ INSIGHTS_BREAKDOWNS_OPTIONS,
239
+ TInsightsBreakdownOptions,
240
+ TInsightsLevels,
241
+ )
242
+
243
+ parts = table.split(":")
244
+
245
+ source_kwargs = {}
246
+
247
+ breakdown_type = parts[1]
248
+
249
+ valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
250
+ if breakdown_type in valid_breakdowns:
251
+ dimensions = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["breakdowns"]
252
+ fields = INSIGHTS_BREAKDOWNS_OPTIONS[breakdown_type]["fields"]
253
+ source_kwargs["dimensions"] = dimensions
254
+ source_kwargs["fields"] = fields
255
+ else:
256
+ dimensions = breakdown_type.split(",")
257
+ valid_levels = list(typing.get_args(TInsightsLevels))
258
+ level = None
259
+ for valid_level in reversed(valid_levels):
260
+ if valid_level in dimensions:
261
+ level = valid_level
262
+ dimensions.remove(valid_level)
263
+ break
264
+
265
+ source_kwargs["level"] = level
266
+ source_kwargs["dimensions"] = dimensions
267
+
268
+ # If custom metrics are provided, parse them
269
+ if len(parts) == 3:
270
+ fields = [f.strip() for f in parts[2].split(",") if f.strip()]
271
+ if not fields:
272
+ raise ValueError(
273
+ "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
274
+ )
275
+ source_kwargs["fields"] = fields
276
+
277
+ return source_kwargs
@@ -3,7 +3,23 @@ from typing import Any, Dict, Iterable, Iterator
3
3
  import dlt
4
4
  import pendulum
5
5
 
6
- from .helpers import _normalize_issue, _normalize_team, _paginate
6
+ from .helpers import _paginate, normalize_dictionaries
7
+
8
+
9
+ def _get_date_range(updated_at, start_date):
10
+ """Extract current start and end dates from incremental state."""
11
+ if updated_at.last_value:
12
+ current_start_date = pendulum.parse(updated_at.last_value)
13
+ else:
14
+ current_start_date = pendulum.parse(start_date)
15
+
16
+ if updated_at.end_value:
17
+ current_end_date = pendulum.parse(updated_at.end_value)
18
+ else:
19
+ current_end_date = pendulum.now(tz="UTC")
20
+
21
+ return current_start_date, current_end_date
22
+
7
23
 
8
24
  ISSUES_QUERY = """
9
25
  query Issues($cursor: String) {
@@ -84,6 +100,25 @@ query Users($cursor: String) {
84
100
  }
85
101
  }
86
102
  """
103
+ WORKFLOW_STATES_QUERY = """
104
+ query WorkflowStates($cursor: String) {
105
+ workflowStates(first: 50, after: $cursor) {
106
+ nodes {
107
+ archivedAt
108
+ color
109
+ createdAt
110
+ id
111
+ inheritedFrom { id }
112
+ name
113
+ position
114
+ team { id }
115
+ type
116
+ updatedAt
117
+ }
118
+ pageInfo { hasNextPage endCursor }
119
+ }
120
+ }
121
+ """
87
122
 
88
123
 
89
124
  @dlt.source(name="linear", max_table_nesting=0)
@@ -102,20 +137,12 @@ def linear_source(
102
137
  range_end="closed",
103
138
  ),
104
139
  ) -> Iterator[Dict[str, Any]]:
105
- if updated_at.last_value:
106
- current_start_date = pendulum.parse(updated_at.last_value)
107
- else:
108
- current_start_date = pendulum.parse(start_date)
109
-
110
- if updated_at.end_value:
111
- current_end_date = pendulum.parse(updated_at.end_value)
112
- else:
113
- current_end_date = pendulum.now(tz="UTC")
140
+ current_start_date, current_end_date = _get_date_range(updated_at, start_date)
114
141
 
115
142
  for item in _paginate(api_key, ISSUES_QUERY, "issues"):
116
143
  if pendulum.parse(item["updatedAt"]) >= current_start_date:
117
144
  if pendulum.parse(item["updatedAt"]) <= current_end_date:
118
- yield _normalize_issue(item)
145
+ yield normalize_dictionaries(item)
119
146
 
120
147
  @dlt.resource(name="projects", primary_key="id", write_disposition="merge")
121
148
  def projects(
@@ -127,20 +154,12 @@ def linear_source(
127
154
  range_end="closed",
128
155
  ),
129
156
  ) -> Iterator[Dict[str, Any]]:
130
- if updated_at.last_value:
131
- current_start_date = pendulum.parse(updated_at.last_value)
132
- else:
133
- current_start_date = pendulum.parse(start_date)
134
-
135
- if updated_at.end_value:
136
- current_end_date = pendulum.parse(updated_at.end_value)
137
- else:
138
- current_end_date = pendulum.now(tz="UTC")
157
+ current_start_date, current_end_date = _get_date_range(updated_at, start_date)
139
158
 
140
159
  for item in _paginate(api_key, PROJECTS_QUERY, "projects"):
141
160
  if pendulum.parse(item["updatedAt"]) >= current_start_date:
142
161
  if pendulum.parse(item["updatedAt"]) <= current_end_date:
143
- yield item
162
+ yield normalize_dictionaries(item)
144
163
 
145
164
  @dlt.resource(name="teams", primary_key="id", write_disposition="merge")
146
165
  def teams(
@@ -153,21 +172,13 @@ def linear_source(
153
172
  ),
154
173
  ) -> Iterator[Dict[str, Any]]:
155
174
  print(start_date)
156
- if updated_at.last_value:
157
- current_start_date = pendulum.parse(updated_at.last_value)
158
- else:
159
- current_start_date = pendulum.parse(start_date)
175
+ current_start_date, current_end_date = _get_date_range(updated_at, start_date)
160
176
  print(current_start_date)
161
177
 
162
- if updated_at.end_value:
163
- current_end_date = pendulum.parse(updated_at.end_value)
164
- else:
165
- current_end_date = pendulum.now(tz="UTC")
166
-
167
178
  for item in _paginate(api_key, TEAMS_QUERY, "teams"):
168
179
  if pendulum.parse(item["updatedAt"]) >= current_start_date:
169
180
  if pendulum.parse(item["updatedAt"]) <= current_end_date:
170
- yield _normalize_team(item)
181
+ yield normalize_dictionaries(item)
171
182
 
172
183
  @dlt.resource(name="users", primary_key="id", write_disposition="merge")
173
184
  def users(
@@ -179,19 +190,28 @@ def linear_source(
179
190
  range_end="closed",
180
191
  ),
181
192
  ) -> Iterator[Dict[str, Any]]:
182
- if updated_at.last_value:
183
- current_start_date = pendulum.parse(updated_at.last_value)
184
- else:
185
- current_start_date = pendulum.parse(start_date)
186
-
187
- if updated_at.end_value:
188
- current_end_date = pendulum.parse(updated_at.end_value)
189
- else:
190
- current_end_date = pendulum.now(tz="UTC")
193
+ current_start_date, current_end_date = _get_date_range(updated_at, start_date)
191
194
 
192
195
  for item in _paginate(api_key, USERS_QUERY, "users"):
193
196
  if pendulum.parse(item["updatedAt"]) >= current_start_date:
194
197
  if pendulum.parse(item["updatedAt"]) <= current_end_date:
195
- yield item
198
+ yield normalize_dictionaries(item)
199
+
200
+ @dlt.resource(name="workflow_states", primary_key="id", write_disposition="merge")
201
+ def workflow_states(
202
+ updated_at: dlt.sources.incremental[str] = dlt.sources.incremental(
203
+ "updatedAt",
204
+ initial_value=start_date.isoformat(),
205
+ end_value=end_date.isoformat() if end_date else None,
206
+ range_start="closed",
207
+ range_end="closed",
208
+ ),
209
+ ) -> Iterator[Dict[str, Any]]:
210
+ current_start_date, current_end_date = _get_date_range(updated_at, start_date)
211
+
212
+ for item in _paginate(api_key, WORKFLOW_STATES_QUERY, "workflowStates"):
213
+ if pendulum.parse(item["updatedAt"]) >= current_start_date:
214
+ if pendulum.parse(item["updatedAt"]) <= current_end_date:
215
+ yield normalize_dictionaries(item)
196
216
 
197
- return issues, projects, teams, users
217
+ return [issues, projects, teams, users, workflow_states]
@@ -32,41 +32,22 @@ def _paginate(api_key: str, query: str, root: str) -> Iterator[Dict[str, Any]]:
32
32
  cursor = data["pageInfo"]["endCursor"]
33
33
 
34
34
 
35
- def _normalize_issue(item: Dict[str, Any]) -> Dict[str, Any]:
36
- field_mapping = {
37
- "assignee": "assignee_id",
38
- "creator": "creator_id",
39
- "state": "state_id",
40
- "cycle": "cycle_id",
41
- "project": "project_id",
42
- }
43
- for key, value in field_mapping.items():
44
- if item.get(key):
45
- item[value] = item[key]["id"]
46
- del item[key]
47
- else:
48
- item[value] = None
49
- del item[key]
50
- json_fields = [
51
- "comments",
52
- "subscribers",
53
- "attachments",
54
- "labels",
55
- "subtasks",
56
- "projects",
57
- "memberships",
58
- "members",
59
- ]
60
- for field in json_fields:
61
- if item.get(field):
62
- item[f"{field}"] = item[field].get("nodes", [])
63
-
64
- return item
65
-
66
-
67
- def _normalize_team(item: Dict[str, Any]) -> Dict[str, Any]:
68
- json_fields = ["memberships", "members", "projects"]
69
- for field in json_fields:
70
- if item.get(field):
71
- item[f"{field}"] = item[field].get("nodes", [])
72
- return item
35
+ def normalize_dictionaries(item: Dict[str, Any]) -> Dict[str, Any]:
36
+ """
37
+ Automatically normalize dictionary fields by detecting their structure:
38
+ - Convert nested objects with 'id' field to {field_name}_id
39
+ - Convert objects with 'nodes' field to arrays
40
+ """
41
+ normalized_item = item.copy()
42
+
43
+ for key, value in list(normalized_item.items()):
44
+ if isinstance(value, dict):
45
+ # If the dict has an 'id' field, replace with {key}_id
46
+ if "id" in value:
47
+ normalized_item[f"{key}_id"] = value["id"]
48
+ del normalized_item[key]
49
+ # If the dict has 'nodes' field, extract the nodes array
50
+ elif "nodes" in value:
51
+ normalized_item[key] = value["nodes"]
52
+
53
+ return normalized_item
ingestr/src/sources.py CHANGED
@@ -702,6 +702,11 @@ class ShopifySource:
702
702
  return True
703
703
 
704
704
  def dlt_source(self, uri: str, table: str, **kwargs):
705
+ if kwargs.get("incremental_key"):
706
+ raise ValueError(
707
+ "Shopify takes care of incrementality on its own, you should not provide incremental_key"
708
+ )
709
+
705
710
  source_fields = urlparse(uri)
706
711
  source_params = parse_qs(source_fields.query)
707
712
  api_key = source_params.get("api_key")
@@ -1003,6 +1008,16 @@ class FacebookAdsSource:
1003
1008
  facebook_insights_source,
1004
1009
  )
1005
1010
 
1011
+ insights_max_wait_to_finish_seconds = source_params.get(
1012
+ "insights_max_wait_to_finish_seconds", [60 * 60 * 4]
1013
+ )
1014
+ insights_max_wait_to_start_seconds = source_params.get(
1015
+ "insights_max_wait_to_start_seconds", [60 * 30]
1016
+ )
1017
+ insights_max_async_sleep_seconds = source_params.get(
1018
+ "insights_max_async_sleep_seconds", [20]
1019
+ )
1020
+
1006
1021
  endpoint = None
1007
1022
  if table in ["campaigns", "ad_sets", "ad_creatives", "ads", "leads"]:
1008
1023
  endpoint = table
@@ -1012,6 +1027,13 @@ class FacebookAdsSource:
1012
1027
  account_id=account_id[0],
1013
1028
  start_date=kwargs.get("interval_start"),
1014
1029
  end_date=kwargs.get("interval_end"),
1030
+ insights_max_wait_to_finish_seconds=insights_max_wait_to_finish_seconds[
1031
+ 0
1032
+ ],
1033
+ insights_max_wait_to_start_seconds=insights_max_wait_to_start_seconds[
1034
+ 0
1035
+ ],
1036
+ insights_max_async_sleep_seconds=insights_max_async_sleep_seconds[0],
1015
1037
  ).with_resources("facebook_insights")
1016
1038
  elif table.startswith("facebook_insights:"):
1017
1039
  # Parse custom breakdowns and metrics from table name
@@ -1034,33 +1056,16 @@ class FacebookAdsSource:
1034
1056
  # Validate breakdown type against available options from settings
1035
1057
  import typing
1036
1058
 
1037
- from ingestr.src.facebook_ads.settings import TInsightsBreakdownOptions
1038
-
1039
- # Get valid breakdown options from the type definition
1040
- valid_breakdowns = list(typing.get_args(TInsightsBreakdownOptions))
1041
-
1042
- if breakdown_type not in valid_breakdowns:
1043
- raise ValueError(
1044
- f"Invalid breakdown type '{breakdown_type}'. Valid options: {', '.join(valid_breakdowns)}"
1045
- )
1059
+ from ingestr.src.facebook_ads.helpers import parse_insights_table_to_source_kwargs
1046
1060
 
1047
1061
  source_kwargs = {
1048
1062
  "access_token": access_token[0],
1049
1063
  "account_id": account_id[0],
1050
1064
  "start_date": kwargs.get("interval_start"),
1051
1065
  "end_date": kwargs.get("interval_end"),
1052
- "breakdowns": breakdown_type,
1053
1066
  }
1054
1067
 
1055
- # If custom metrics are provided, parse them
1056
- if len(parts) == 3:
1057
- fields = [f.strip() for f in parts[2].split(",") if f.strip()]
1058
- if not fields:
1059
- raise ValueError(
1060
- "Custom metrics must be provided after the second colon in format: facebook_insights:breakdown_type:metric1,metric2..."
1061
- )
1062
- source_kwargs["fields"] = fields
1063
-
1068
+ source_kwargs.update(parse_insights_table_to_source_kwargs(table))
1064
1069
  return facebook_insights_source(**source_kwargs).with_resources(
1065
1070
  "facebook_insights"
1066
1071
  )
@@ -1652,6 +1657,11 @@ class TikTokSource:
1652
1657
  return True
1653
1658
 
1654
1659
  def dlt_source(self, uri: str, table: str, **kwargs):
1660
+ if kwargs.get("incremental_key"):
1661
+ raise ValueError(
1662
+ "TikTok takes care of incrementality on its own, you should not provide incremental_key"
1663
+ )
1664
+
1655
1665
  endpoint = "custom_reports"
1656
1666
 
1657
1667
  parsed_uri = urlparse(uri)
@@ -1874,6 +1884,11 @@ class GoogleAnalyticsSource:
1874
1884
  def dlt_source(self, uri: str, table: str, **kwargs):
1875
1885
  import ingestr.src.google_analytics.helpers as helpers
1876
1886
 
1887
+ if kwargs.get("incremental_key"):
1888
+ raise ValueError(
1889
+ "Google Analytics takes care of incrementality on its own, you should not provide incremental_key"
1890
+ )
1891
+
1877
1892
  result = helpers.parse_google_analytics_uri(uri)
1878
1893
  credentials = result["credentials"]
1879
1894
  property_id = result["property_id"]
@@ -2246,6 +2261,11 @@ class LinkedInAdsSource:
2246
2261
  return True
2247
2262
 
2248
2263
  def dlt_source(self, uri: str, table: str, **kwargs):
2264
+ if kwargs.get("incremental_key"):
2265
+ raise ValueError(
2266
+ "LinkedIn Ads takes care of incrementality on its own, you should not provide incremental_key"
2267
+ )
2268
+
2249
2269
  parsed_uri = urlparse(uri)
2250
2270
  source_fields = parse_qs(parsed_uri.query)
2251
2271
 
@@ -2329,6 +2349,11 @@ class ClickupSource:
2329
2349
  return True
2330
2350
 
2331
2351
  def dlt_source(self, uri: str, table: str, **kwargs):
2352
+ if kwargs.get("incremental_key"):
2353
+ raise ValueError(
2354
+ "ClickUp takes care of incrementality on its own, you should not provide incremental_key"
2355
+ )
2356
+
2332
2357
  parsed_uri = urlparse(uri)
2333
2358
  params = parse_qs(parsed_uri.query)
2334
2359
  api_token = params.get("api_token")
@@ -2413,6 +2438,11 @@ class ApplovinMaxSource:
2413
2438
  return True
2414
2439
 
2415
2440
  def dlt_source(self, uri: str, table: str, **kwargs):
2441
+ if kwargs.get("incremental_key"):
2442
+ raise ValueError(
2443
+ "AppLovin Max takes care of incrementality on its own, you should not provide incremental_key"
2444
+ )
2445
+
2416
2446
  parsed_uri = urlparse(uri)
2417
2447
  params = parse_qs(parsed_uri.query)
2418
2448
 
@@ -2505,6 +2535,11 @@ class PersonioSource:
2505
2535
 
2506
2536
  # applovin://?client_id=123&client_secret=123
2507
2537
  def dlt_source(self, uri: str, table: str, **kwargs):
2538
+ if kwargs.get("incremental_key"):
2539
+ raise ValueError(
2540
+ "Personio takes care of incrementality on its own, you should not provide incremental_key"
2541
+ )
2542
+
2508
2543
  parsed_uri = urlparse(uri)
2509
2544
  params = parse_qs(parsed_uri.query)
2510
2545
 
@@ -2595,6 +2630,11 @@ class PipedriveSource:
2595
2630
  return True
2596
2631
 
2597
2632
  def dlt_source(self, uri: str, table: str, **kwargs):
2633
+ if kwargs.get("incremental_key"):
2634
+ raise ValueError(
2635
+ "Pipedrive takes care of incrementality on its own, you should not provide incremental_key"
2636
+ )
2637
+
2598
2638
  parsed_uri = urlparse(uri)
2599
2639
  params = parse_qs(parsed_uri.query)
2600
2640
  api_key = params.get("api_token")
@@ -2677,6 +2717,11 @@ class FreshdeskSource:
2677
2717
  return True
2678
2718
 
2679
2719
  def dlt_source(self, uri: str, table: str, **kwargs):
2720
+ if kwargs.get("incremental_key"):
2721
+ raise ValueError(
2722
+ "Freshdesk takes care of incrementality on its own, you should not provide incremental_key"
2723
+ )
2724
+
2680
2725
  parsed_uri = urlparse(uri)
2681
2726
  domain = parsed_uri.netloc
2682
2727
  query = parsed_uri.query
@@ -2730,6 +2775,11 @@ class TrustpilotSource:
2730
2775
  return True
2731
2776
 
2732
2777
  def dlt_source(self, uri: str, table: str, **kwargs):
2778
+ if kwargs.get("incremental_key"):
2779
+ raise ValueError(
2780
+ "Trustpilot takes care of incrementality on its own, you should not provide incremental_key"
2781
+ )
2782
+
2733
2783
  parsed_uri = urlparse(uri)
2734
2784
  business_unit_id = parsed_uri.netloc
2735
2785
  params = parse_qs(parsed_uri.query)
@@ -2770,6 +2820,11 @@ class PhantombusterSource:
2770
2820
  return True
2771
2821
 
2772
2822
  def dlt_source(self, uri: str, table: str, **kwargs):
2823
+ if kwargs.get("incremental_key"):
2824
+ raise ValueError(
2825
+ "Phantombuster takes care of incrementality on its own, you should not provide incremental_key"
2826
+ )
2827
+
2773
2828
  # phantombuster://?api_key=<api_key>
2774
2829
  # source table = phantom_results:agent_id
2775
2830
  parsed_uri = urlparse(uri)
@@ -2923,6 +2978,11 @@ class SolidgateSource:
2923
2978
  return True
2924
2979
 
2925
2980
  def dlt_source(self, uri: str, table: str, **kwargs):
2981
+ if kwargs.get("incremental_key"):
2982
+ raise ValueError(
2983
+ "Solidgate takes care of incrementality on its own, you should not provide incremental_key"
2984
+ )
2985
+
2926
2986
  parsed_uri = urlparse(uri)
2927
2987
  query_params = parse_qs(parsed_uri.query)
2928
2988
  public_key = query_params.get("public_key")
@@ -3016,6 +3076,11 @@ class QuickBooksSource:
3016
3076
 
3017
3077
  # quickbooks://?company_id=<company_id>&client_id=<client_id>&client_secret=<client_secret>&refresh_token=<refresh>&access_token=<access_token>&environment=<env>&minor_version=<version>
3018
3078
  def dlt_source(self, uri: str, table: str, **kwargs):
3079
+ if kwargs.get("incremental_key"):
3080
+ raise ValueError(
3081
+ "QuickBooks takes care of incrementality on its own, you should not provide incremental_key"
3082
+ )
3083
+
3019
3084
  parsed_uri = urlparse(uri)
3020
3085
 
3021
3086
  params = parse_qs(parsed_uri.query)
@@ -3085,6 +3150,11 @@ class IsocPulseSource:
3085
3150
  return True
3086
3151
 
3087
3152
  def dlt_source(self, uri: str, table: str, **kwargs):
3153
+ if kwargs.get("incremental_key"):
3154
+ raise ValueError(
3155
+ "Internet Society Pulse takes care of incrementality on its own, you should not provide incremental_key"
3156
+ )
3157
+
3088
3158
  parsed_uri = urlparse(uri)
3089
3159
  params = parse_qs(parsed_uri.query)
3090
3160
  token = params.get("token")
@@ -3120,6 +3190,11 @@ class PinterestSource:
3120
3190
  return True
3121
3191
 
3122
3192
  def dlt_source(self, uri: str, table: str, **kwargs):
3193
+ if kwargs.get("incremental_key"):
3194
+ raise ValueError(
3195
+ "Pinterest takes care of incrementality on its own, you should not provide incremental_key"
3196
+ )
3197
+
3123
3198
  parsed = urlparse(uri)
3124
3199
  params = parse_qs(parsed.query)
3125
3200
  access_token = params.get("access_token")
@@ -3154,13 +3229,18 @@ class LinearSource:
3154
3229
  return True
3155
3230
 
3156
3231
  def dlt_source(self, uri: str, table: str, **kwargs):
3232
+ if kwargs.get("incremental_key"):
3233
+ raise ValueError(
3234
+ "Linear takes care of incrementality on its own, you should not provide incremental_key"
3235
+ )
3236
+
3157
3237
  parsed_uri = urlparse(uri)
3158
3238
  params = parse_qs(parsed_uri.query)
3159
3239
  api_key = params.get("api_key")
3160
3240
  if api_key is None:
3161
3241
  raise MissingValueError("api_key", "Linear")
3162
3242
 
3163
- if table not in ["issues", "projects", "teams", "users"]:
3243
+ if table not in ["issues", "projects", "teams", "users", "workflow_states"]:
3164
3244
  raise UnsupportedResourceError(table, "Linear")
3165
3245
 
3166
3246
  start_date = kwargs.get("interval_start")
@@ -3187,6 +3267,11 @@ class ZoomSource:
3187
3267
  return True
3188
3268
 
3189
3269
  def dlt_source(self, uri: str, table: str, **kwargs):
3270
+ if kwargs.get("incremental_key"):
3271
+ raise ValueError(
3272
+ "Zoom takes care of incrementality on its own, you should not provide incremental_key"
3273
+ )
3274
+
3190
3275
  parsed = urlparse(uri)
3191
3276
  params = parse_qs(parsed.query)
3192
3277
  client_id = params.get("client_id")
@@ -3228,6 +3313,11 @@ class InfluxDBSource:
3228
3313
  return True
3229
3314
 
3230
3315
  def dlt_source(self, uri: str, table: str, **kwargs):
3316
+ if kwargs.get("incremental_key"):
3317
+ raise ValueError(
3318
+ "InfluxDB takes care of incrementality on its own, you should not provide incremental_key"
3319
+ )
3320
+
3231
3321
  parsed_uri = urlparse(uri)
3232
3322
  params = parse_qs(parsed_uri.query)
3233
3323
  host = parsed_uri.hostname
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: ingestr
3
- Version: 0.13.79
3
+ Version: 0.13.81
4
4
  Summary: ingestr is a command-line application that ingests data from various sources and stores them in any database.
5
5
  Project-URL: Homepage, https://github.com/bruin-data/ingestr
6
6
  Project-URL: Issues, https://github.com/bruin-data/ingestr/issues
@@ -2,7 +2,7 @@ ingestr/conftest.py,sha256=OE2yxeTCosS9CUFVuqNypm-2ftYvVBeeq7egm3878cI,1981
2
2
  ingestr/main.py,sha256=qoWHNcHh0-xVnyQxbQ-SKuTxPb1RNV3ENkCpqO7CLrk,26694
3
3
  ingestr/src/.gitignore,sha256=8cX1AZTSI0TcdZFGTmS_oyBjpfCzhOEt0DdAo2dFIY8,203
4
4
  ingestr/src/blob.py,sha256=UUWMjHUuoR9xP1XZQ6UANQmnMVyDx3d0X4-2FQC271I,2138
5
- ingestr/src/buildinfo.py,sha256=yE0cfxWae8TNJJLYcRmNexeK769vtdz_-vJGzcROgwE,21
5
+ ingestr/src/buildinfo.py,sha256=H6wsXWMjlzTHcGuLKaXl67mkAFCFJzin5gNxWfo6nMg,21
6
6
  ingestr/src/destinations.py,sha256=M2Yni6wiWcrvZ8EPJemidqxN156l0rehgCc7xuil7mo,22840
7
7
  ingestr/src/errors.py,sha256=Ufs4_DfE77_E3vnA1fOQdi6cmuLVNm7_SbFLkL1XPGk,686
8
8
  ingestr/src/factory.py,sha256=rF5Ry4o4t8KulSPBtrd7ZKCI_0TH1DAetG0zs9H7oik,6792
@@ -11,7 +11,7 @@ ingestr/src/http_client.py,sha256=bxqsk6nJNXCo-79gW04B53DQO-yr25vaSsqP0AKtjx4,73
11
11
  ingestr/src/loader.py,sha256=9NaWAyfkXdqAZSS-N72Iwo36Lbx4PyqIfaaH1dNdkFs,1712
12
12
  ingestr/src/partition.py,sha256=BrIP6wFJvyR7Nus_3ElnfxknUXeCipK_E_bB8kZowfc,969
13
13
  ingestr/src/resource.py,sha256=ZqmZxFQVGlF8rFPhBiUB08HES0yoTj8sZ--jKfaaVps,1164
14
- ingestr/src/sources.py,sha256=qZz35cdO-nO9CZsdOJ8Ni56wclNfbGQuGj4nsoHpFxE,115678
14
+ ingestr/src/sources.py,sha256=wdNRqVzuR8EK8s8SzL3lZ8UE6UrqLMq-FJGkc23rrpM,119190
15
15
  ingestr/src/table_definition.py,sha256=REbAbqdlmUMUuRh8nEQRreWjPVOQ5ZcfqGkScKdCrmk,390
16
16
  ingestr/src/time.py,sha256=H_Fk2J4ShXyUM-EMY7MqCLZQhlnZMZvO952bmZPc4yE,254
17
17
  ingestr/src/version.py,sha256=J_2xgZ0mKlvuHcjdKCx2nlioneLH0I47JiU_Slr_Nwc,189
@@ -41,9 +41,9 @@ ingestr/src/clickup/helpers.py,sha256=RzDKMUAHccuDhocIQ2ToBXfCERo8CBJqA3t-IPltBC
41
41
  ingestr/src/collector/spinner.py,sha256=_ZUqF5MI43hVIULdjF5s5mrAZbhEFXaiWirQmrv3Yk4,1201
42
42
  ingestr/src/dynamodb/__init__.py,sha256=swhxkeYBbJ35jn1IghCtvYWT2BM33KynVCh_oR4z28A,2264
43
43
  ingestr/src/elasticsearch/__init__.py,sha256=m-q93HgUmTwGDUwHOjHawstWL06TC3WIX3H05szybrY,2556
44
- ingestr/src/facebook_ads/__init__.py,sha256=_9929DYzcq5iLt-l3DmJ4VBZwmoEwgyPZbPstH0ySmI,9725
44
+ ingestr/src/facebook_ads/__init__.py,sha256=15GiovITANe0al5MI6WWLdl3LDmdBd1YpkUWBV3g6bk,9715
45
45
  ingestr/src/facebook_ads/exceptions.py,sha256=4Nlbc0Mv3i5g-9AoyT-n1PIa8IDi3VCTfEAzholx4Wc,115
46
- ingestr/src/facebook_ads/helpers.py,sha256=NshS21can1xhRKQzg_o-c6qSxWoC3NnE3FwgJxUnygE,8239
46
+ ingestr/src/facebook_ads/helpers.py,sha256=c-WG008yU_zIdhFwljtqE2jfjVYuaVoNKldxcnJN3U4,9761
47
47
  ingestr/src/facebook_ads/settings.py,sha256=Bsic8RcmH-NfEZ7r_NGospTCmwISK9XaMT5y2NZirtg,4938
48
48
  ingestr/src/facebook_ads/utils.py,sha256=ES2ylPoW3j3fjp6OMUgp21n1cG1OktXsmWWMk5vBW_I,1590
49
49
  ingestr/src/filesystem/__init__.py,sha256=zkIwbRr0ir0EUdniI25p2zGiVc-7M9EmR351AjNb0eA,4163
@@ -85,8 +85,8 @@ ingestr/src/kinesis/helpers.py,sha256=SO2cFmWNGcykUYmjHdfxWsOQSkLQXyhFtfWnkcUOM0
85
85
  ingestr/src/klaviyo/__init__.py,sha256=o_noUgbxLk36s4f9W56_ibPorF0n7kVapPUlV0p-jfA,7875
86
86
  ingestr/src/klaviyo/client.py,sha256=tPj79ia7AW0ZOJhzlKNPCliGbdojRNwUFp8HvB2ym5s,7434
87
87
  ingestr/src/klaviyo/helpers.py,sha256=_i-SHffhv25feLDcjy6Blj1UxYLISCwVCMgGtrlnYHk,496
88
- ingestr/src/linear/__init__.py,sha256=Qbf8EPHVh-8pVNe_fqLVinds7qQ3O4ymDuPPPD560Ng,5953
89
- ingestr/src/linear/helpers.py,sha256=Mb7oKpUTRnHl-CvO1fubjJlJFDkhTuA7PUldWglvagI,2044
88
+ ingestr/src/linear/__init__.py,sha256=CZnRQfP44L1fy4ppnukuTr1wWKSNlPpaLFE3Xw5vJr0,6605
89
+ ingestr/src/linear/helpers.py,sha256=jpG9l3JhJVXk2K77iny8BMqkYHGycwS4Kmcdq2Xs7aE,1786
90
90
  ingestr/src/linkedin_ads/__init__.py,sha256=CAPWFyV24loziiphbLmODxZUXZJwm4JxlFkr56q0jfo,1855
91
91
  ingestr/src/linkedin_ads/dimension_time_enum.py,sha256=EmHRdkFyTAfo4chGjThrwqffWJxmAadZMbpTvf0xkQc,198
92
92
  ingestr/src/linkedin_ads/helpers.py,sha256=eUWudRVlXl4kqIhfXQ1eVsUpZwJn7UFqKSpnbLfxzds,4498
@@ -151,8 +151,8 @@ ingestr/testdata/merge_expected.csv,sha256=DReHqWGnQMsf2PBv_Q2pfjsgvikYFnf1zYcQZ
151
151
  ingestr/testdata/merge_part1.csv,sha256=Pw8Z9IDKcNU0qQHx1z6BUf4rF_-SxKGFOvymCt4OY9I,185
152
152
  ingestr/testdata/merge_part2.csv,sha256=T_GiWxA81SN63_tMOIuemcvboEFeAmbKc7xRXvL9esw,287
153
153
  ingestr/tests/unit/test_smartsheets.py,sha256=eiC2CCO4iNJcuN36ONvqmEDryCA1bA1REpayHpu42lk,5058
154
- ingestr-0.13.79.dist-info/METADATA,sha256=5dl0NFB3Ach1_lFtE4xOJpud_chn_w0qvepZnnMjRzo,15182
155
- ingestr-0.13.79.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
156
- ingestr-0.13.79.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
157
- ingestr-0.13.79.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
158
- ingestr-0.13.79.dist-info/RECORD,,
154
+ ingestr-0.13.81.dist-info/METADATA,sha256=jjolsfHDoVMclLreJMhKFu6s1_FZ5d_eN2F8CFgEDFA,15182
155
+ ingestr-0.13.81.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
156
+ ingestr-0.13.81.dist-info/entry_points.txt,sha256=oPJy0KBnPWYjDtP1k8qwAihcTLHSZokSQvRAw_wtfJM,46
157
+ ingestr-0.13.81.dist-info/licenses/LICENSE.md,sha256=cW8wIhn8HFE-KLStDF9jHQ1O_ARWP3kTpk_-eOccL24,1075
158
+ ingestr-0.13.81.dist-info/RECORD,,