airbyte-source-iterable 0.7.0.dev202603131157__tar.gz → 0.7.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/PKG-INFO +1 -1
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/pyproject.toml +1 -1
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/manifest.yaml +22 -20
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/slice_generators.py +3 -1
- airbyte_source_iterable-0.7.1/source_iterable/source.py +116 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/spec.json +7 -6
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/streams.py +22 -13
- airbyte_source_iterable-0.7.0.dev202603131157/source_iterable/source.py +0 -114
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/README.md +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/__init__.py +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/components.py +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/run.py +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/campaigns.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/campaigns_metrics.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/channels.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_bounce.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_click.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_complaint.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_open.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_send.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_send_skip.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_subscribe.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/email_unsubscribe.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/events.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/list_users.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/lists.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/message_types.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/metadata.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/templates.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/schemas/users.json +0 -0
- {airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/source_iterable/utils.py +0 -0
{airbyte_source_iterable-0.7.0.dev202603131157 → airbyte_source_iterable-0.7.1}/pyproject.toml
RENAMED
|
@@ -3,7 +3,7 @@ requires = [ "poetry-core>=1.0.0",]
|
|
|
3
3
|
build-backend = "poetry.core.masonry.api"
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
|
-
version = "0.7.
|
|
6
|
+
version = "0.7.1"
|
|
7
7
|
name = "airbyte-source-iterable"
|
|
8
8
|
description = "Source implementation for Iterable."
|
|
9
9
|
authors = [ "Airbyte <contact@airbyte.io>",]
|
|
@@ -21,21 +21,23 @@ spec:
|
|
|
21
21
|
type: "string"
|
|
22
22
|
title: "Start Date"
|
|
23
23
|
description: >-
|
|
24
|
-
The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.
|
|
24
|
+
The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.
|
|
25
25
|
All data generated after this date will be replicated.
|
|
26
26
|
examples: ["2021-04-01T00:00:00Z"]
|
|
27
27
|
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
|
|
28
28
|
order: 1
|
|
29
29
|
format: "date-time"
|
|
30
|
-
|
|
31
|
-
type: "
|
|
32
|
-
title: "
|
|
30
|
+
lookback_window:
|
|
31
|
+
type: "integer"
|
|
32
|
+
title: "Lookback Window (Minutes)"
|
|
33
33
|
description: >-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
default:
|
|
34
|
+
Number of minutes to re-read from the current time when determining the end of each sync window
|
|
35
|
+
for export-based streams. This accounts for eventual consistency delays in Iterable's Export API,
|
|
36
|
+
preventing silent data loss for events not yet indexed. Increase this value if you observe missing
|
|
37
|
+
events near the end of sync windows.
|
|
38
|
+
default: 5
|
|
39
|
+
minimum: 0
|
|
40
|
+
examples: [5, 10, 15]
|
|
39
41
|
order: 2
|
|
40
42
|
type: DeclarativeSource
|
|
41
43
|
check:
|
|
@@ -52,7 +54,7 @@ streams:
|
|
|
52
54
|
requester:
|
|
53
55
|
path: lists
|
|
54
56
|
type: HttpRequester
|
|
55
|
-
url_base:
|
|
57
|
+
url_base: https://api.iterable.com/api/
|
|
56
58
|
http_method: GET
|
|
57
59
|
authenticator:
|
|
58
60
|
type: ApiKeyAuthenticator
|
|
@@ -84,7 +86,7 @@ streams:
|
|
|
84
86
|
requester:
|
|
85
87
|
path: lists/getUsers
|
|
86
88
|
type: HttpRequester
|
|
87
|
-
url_base:
|
|
89
|
+
url_base: https://api.iterable.com/api/
|
|
88
90
|
http_method: GET
|
|
89
91
|
authenticator:
|
|
90
92
|
type: ApiKeyAuthenticator
|
|
@@ -120,7 +122,7 @@ streams:
|
|
|
120
122
|
type: SimpleRetriever
|
|
121
123
|
requester:
|
|
122
124
|
type: HttpRequester
|
|
123
|
-
url_base:
|
|
125
|
+
url_base: https://api.iterable.com/api/
|
|
124
126
|
path: lists
|
|
125
127
|
http_method: GET
|
|
126
128
|
request_parameters: {}
|
|
@@ -165,7 +167,7 @@ streams:
|
|
|
165
167
|
requester:
|
|
166
168
|
path: campaigns
|
|
167
169
|
type: HttpRequester
|
|
168
|
-
url_base:
|
|
170
|
+
url_base: https://api.iterable.com/api/
|
|
169
171
|
http_method: GET
|
|
170
172
|
authenticator:
|
|
171
173
|
type: ApiKeyAuthenticator
|
|
@@ -195,7 +197,7 @@ streams:
|
|
|
195
197
|
requester:
|
|
196
198
|
path: channels
|
|
197
199
|
type: HttpRequester
|
|
198
|
-
url_base:
|
|
200
|
+
url_base: https://api.iterable.com/api/
|
|
199
201
|
http_method: GET
|
|
200
202
|
authenticator:
|
|
201
203
|
type: ApiKeyAuthenticator
|
|
@@ -225,7 +227,7 @@ streams:
|
|
|
225
227
|
requester:
|
|
226
228
|
path: messageTypes
|
|
227
229
|
type: HttpRequester
|
|
228
|
-
url_base:
|
|
230
|
+
url_base: https://api.iterable.com/api/
|
|
229
231
|
http_method: GET
|
|
230
232
|
authenticator:
|
|
231
233
|
type: ApiKeyAuthenticator
|
|
@@ -255,7 +257,7 @@ streams:
|
|
|
255
257
|
requester:
|
|
256
258
|
path: metadata
|
|
257
259
|
type: HttpRequester
|
|
258
|
-
url_base:
|
|
260
|
+
url_base: https://api.iterable.com/api/
|
|
259
261
|
http_method: GET
|
|
260
262
|
authenticator:
|
|
261
263
|
type: ApiKeyAuthenticator
|
|
@@ -286,7 +288,7 @@ streams:
|
|
|
286
288
|
requester:
|
|
287
289
|
path: export/data.json
|
|
288
290
|
type: HttpRequester
|
|
289
|
-
url_base:
|
|
291
|
+
url_base: https://api.iterable.com/api/
|
|
290
292
|
http_method: GET
|
|
291
293
|
authenticator:
|
|
292
294
|
type: ApiKeyAuthenticator
|
|
@@ -340,7 +342,7 @@ streams:
|
|
|
340
342
|
type: JsonlDecoder
|
|
341
343
|
requester:
|
|
342
344
|
type: HttpRequester
|
|
343
|
-
url_base:
|
|
345
|
+
url_base: https://api.iterable.com/api/
|
|
344
346
|
path: export/userEvents
|
|
345
347
|
http_method: GET
|
|
346
348
|
request_parameters:
|
|
@@ -383,7 +385,7 @@ streams:
|
|
|
383
385
|
requester:
|
|
384
386
|
path: lists/getUsers
|
|
385
387
|
type: HttpRequester
|
|
386
|
-
url_base:
|
|
388
|
+
url_base: https://api.iterable.com/api/
|
|
387
389
|
http_method: GET
|
|
388
390
|
authenticator:
|
|
389
391
|
type: ApiKeyAuthenticator
|
|
@@ -419,7 +421,7 @@ streams:
|
|
|
419
421
|
type: SimpleRetriever
|
|
420
422
|
requester:
|
|
421
423
|
type: HttpRequester
|
|
422
|
-
url_base:
|
|
424
|
+
url_base: https://api.iterable.com/api/
|
|
423
425
|
path: lists
|
|
424
426
|
http_method: GET
|
|
425
427
|
request_parameters: {}
|
|
@@ -108,6 +108,7 @@ class AdjustableSliceGenerator(SliceGenerator):
|
|
|
108
108
|
|
|
109
109
|
REQUEST_PER_MINUTE_LIMIT = 4
|
|
110
110
|
INITIAL_RANGE_DAYS: int = 30
|
|
111
|
+
MIN_RANGE_DAYS: int = 1
|
|
111
112
|
DEFAULT_RANGE_DAYS: int = 90
|
|
112
113
|
MAX_RANGE_DAYS: int = 180
|
|
113
114
|
RANGE_REDUCE_FACTOR = 2
|
|
@@ -145,8 +146,9 @@ class AdjustableSliceGenerator(SliceGenerator):
|
|
|
145
146
|
RANGE_REDUCE_FACTOR (2 times).
|
|
146
147
|
Returns updated slice to try again.
|
|
147
148
|
"""
|
|
148
|
-
self._current_range = int(max(self._current_range / self.RANGE_REDUCE_FACTOR, self.INITIAL_RANGE_DAYS))
|
|
149
149
|
start_date = self._prev_start_date
|
|
150
|
+
actual_range = (self._start_date - start_date).total_seconds() / 86400
|
|
151
|
+
self._current_range = int(max(actual_range / self.RANGE_REDUCE_FACTOR, self.MIN_RANGE_DAYS))
|
|
150
152
|
end_date = min(self._end_date, start_date + (pendulum.Duration(days=self._current_range)))
|
|
151
153
|
self._start_date = end_date
|
|
152
154
|
return StreamSlice(start_date=start_date, end_date=end_date)
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from typing import Any, List, Mapping
|
|
6
|
+
|
|
7
|
+
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
|
8
|
+
from airbyte_cdk.sources.streams import Stream
|
|
9
|
+
from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
|
|
10
|
+
|
|
11
|
+
from .streams import (
|
|
12
|
+
CampaignsMetrics,
|
|
13
|
+
CustomEvent,
|
|
14
|
+
EmailBounce,
|
|
15
|
+
EmailClick,
|
|
16
|
+
EmailComplaint,
|
|
17
|
+
EmailOpen,
|
|
18
|
+
EmailSend,
|
|
19
|
+
EmailSendSkip,
|
|
20
|
+
EmailSubscribe,
|
|
21
|
+
EmailUnsubscribe,
|
|
22
|
+
HostedUnsubscribeClick,
|
|
23
|
+
InAppClick,
|
|
24
|
+
InAppClose,
|
|
25
|
+
InAppDelete,
|
|
26
|
+
InAppDelivery,
|
|
27
|
+
InAppOpen,
|
|
28
|
+
InAppSend,
|
|
29
|
+
InAppSendSkip,
|
|
30
|
+
InboxMessageImpression,
|
|
31
|
+
InboxSession,
|
|
32
|
+
Purchase,
|
|
33
|
+
PushBounce,
|
|
34
|
+
PushOpen,
|
|
35
|
+
PushSend,
|
|
36
|
+
PushSendSkip,
|
|
37
|
+
PushUninstall,
|
|
38
|
+
SmsBounce,
|
|
39
|
+
SmsClick,
|
|
40
|
+
SmsReceived,
|
|
41
|
+
SmsSend,
|
|
42
|
+
SmsSendSkip,
|
|
43
|
+
SmsUsageInfo,
|
|
44
|
+
Templates,
|
|
45
|
+
WebPushClick,
|
|
46
|
+
WebPushSend,
|
|
47
|
+
WebPushSendSkip,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
"""
|
|
52
|
+
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
|
|
53
|
+
source connector.
|
|
54
|
+
|
|
55
|
+
WARNING: Do not modify this file.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Declarative Source
|
|
60
|
+
class SourceIterable(YamlDeclarativeSource):
|
|
61
|
+
def __init__(self):
|
|
62
|
+
super().__init__(**{"path_to_yaml": "manifest.yaml"})
|
|
63
|
+
|
|
64
|
+
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
|
65
|
+
streams = super().streams(config=config)
|
|
66
|
+
|
|
67
|
+
authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
|
|
68
|
+
# end date is provided for integration tests only
|
|
69
|
+
start_date, end_date = config["start_date"], config.get("end_date")
|
|
70
|
+
lookback_window = config.get("lookback_window", 5)
|
|
71
|
+
date_range = {"start_date": start_date, "end_date": end_date, "lookback_window": lookback_window}
|
|
72
|
+
# CampaignsMetrics extends IterableStream (not IterableExportStream) and does not accept lookback_window
|
|
73
|
+
campaigns_date_range = {"start_date": start_date, "end_date": end_date}
|
|
74
|
+
|
|
75
|
+
# TODO: migrate streams below to low code as slicer logic will be migrated to generator based
|
|
76
|
+
streams.extend(
|
|
77
|
+
[
|
|
78
|
+
CampaignsMetrics(authenticator=authenticator, **campaigns_date_range),
|
|
79
|
+
Templates(authenticator=authenticator, **date_range),
|
|
80
|
+
EmailBounce(authenticator=authenticator, **date_range),
|
|
81
|
+
EmailClick(authenticator=authenticator, **date_range),
|
|
82
|
+
EmailComplaint(authenticator=authenticator, **date_range),
|
|
83
|
+
EmailOpen(authenticator=authenticator, **date_range),
|
|
84
|
+
EmailSend(authenticator=authenticator, **date_range),
|
|
85
|
+
EmailSendSkip(authenticator=authenticator, **date_range),
|
|
86
|
+
EmailSubscribe(authenticator=authenticator, **date_range),
|
|
87
|
+
EmailUnsubscribe(authenticator=authenticator, **date_range),
|
|
88
|
+
PushSend(authenticator=authenticator, **date_range),
|
|
89
|
+
PushSendSkip(authenticator=authenticator, **date_range),
|
|
90
|
+
PushOpen(authenticator=authenticator, **date_range),
|
|
91
|
+
PushUninstall(authenticator=authenticator, **date_range),
|
|
92
|
+
PushBounce(authenticator=authenticator, **date_range),
|
|
93
|
+
WebPushSend(authenticator=authenticator, **date_range),
|
|
94
|
+
WebPushClick(authenticator=authenticator, **date_range),
|
|
95
|
+
WebPushSendSkip(authenticator=authenticator, **date_range),
|
|
96
|
+
InAppSend(authenticator=authenticator, **date_range),
|
|
97
|
+
InAppOpen(authenticator=authenticator, **date_range),
|
|
98
|
+
InAppClick(authenticator=authenticator, **date_range),
|
|
99
|
+
InAppClose(authenticator=authenticator, **date_range),
|
|
100
|
+
InAppDelete(authenticator=authenticator, **date_range),
|
|
101
|
+
InAppDelivery(authenticator=authenticator, **date_range),
|
|
102
|
+
InAppSendSkip(authenticator=authenticator, **date_range),
|
|
103
|
+
InboxSession(authenticator=authenticator, **date_range),
|
|
104
|
+
InboxMessageImpression(authenticator=authenticator, **date_range),
|
|
105
|
+
SmsSend(authenticator=authenticator, **date_range),
|
|
106
|
+
SmsBounce(authenticator=authenticator, **date_range),
|
|
107
|
+
SmsClick(authenticator=authenticator, **date_range),
|
|
108
|
+
SmsReceived(authenticator=authenticator, **date_range),
|
|
109
|
+
SmsSendSkip(authenticator=authenticator, **date_range),
|
|
110
|
+
SmsUsageInfo(authenticator=authenticator, **date_range),
|
|
111
|
+
Purchase(authenticator=authenticator, **date_range),
|
|
112
|
+
CustomEvent(authenticator=authenticator, **date_range),
|
|
113
|
+
HostedUnsubscribeClick(authenticator=authenticator, **date_range),
|
|
114
|
+
]
|
|
115
|
+
)
|
|
116
|
+
return streams
|
|
@@ -22,12 +22,13 @@
|
|
|
22
22
|
"order": 1,
|
|
23
23
|
"format": "date-time"
|
|
24
24
|
},
|
|
25
|
-
"
|
|
26
|
-
"type": "
|
|
27
|
-
"title": "
|
|
28
|
-
"description": "
|
|
29
|
-
"
|
|
30
|
-
"
|
|
25
|
+
"lookback_window": {
|
|
26
|
+
"type": "integer",
|
|
27
|
+
"title": "Lookback Window (Minutes)",
|
|
28
|
+
"description": "Number of minutes to re-read from the current time when determining the end of each sync window for export-based streams. This accounts for eventual consistency delays in Iterable's Export API, preventing silent data loss for events not yet indexed. Increase this value if you observe missing events near the end of sync windows.",
|
|
29
|
+
"default": 5,
|
|
30
|
+
"minimum": 0,
|
|
31
|
+
"examples": [5, 10, 15],
|
|
31
32
|
"order": 2
|
|
32
33
|
}
|
|
33
34
|
}
|
|
@@ -33,20 +33,14 @@ class IterableStream(HttpStream, ABC):
|
|
|
33
33
|
# to prevent 429 error on other streams
|
|
34
34
|
ignore_further_slices = False
|
|
35
35
|
|
|
36
|
+
url_base = "https://api.iterable.com/api/"
|
|
36
37
|
primary_key = "id"
|
|
37
38
|
|
|
38
|
-
def __init__(self, authenticator
|
|
39
|
+
def __init__(self, authenticator):
|
|
39
40
|
self._cred = authenticator
|
|
40
41
|
self._slice_retry = 0
|
|
41
|
-
self._region = region
|
|
42
42
|
super().__init__(authenticator)
|
|
43
43
|
|
|
44
|
-
@property
|
|
45
|
-
def url_base(self) -> str:
|
|
46
|
-
if self._region == "EU":
|
|
47
|
-
return "https://api.eu.iterable.com/api/"
|
|
48
|
-
return "https://api.iterable.com/api/"
|
|
49
|
-
|
|
50
44
|
@property
|
|
51
45
|
def retry_factor(self) -> int:
|
|
52
46
|
return 20
|
|
@@ -159,10 +153,11 @@ class IterableExportStream(IterableStream, CheckpointMixin, ABC):
|
|
|
159
153
|
def state(self, value: MutableMapping[str, Any]):
|
|
160
154
|
self._state = value
|
|
161
155
|
|
|
162
|
-
def __init__(self, start_date=None, end_date=None, **kwargs):
|
|
156
|
+
def __init__(self, start_date=None, end_date=None, lookback_window: int = 5, **kwargs):
|
|
163
157
|
super().__init__(**kwargs)
|
|
164
158
|
self._start_date = pendulum.parse(start_date)
|
|
165
159
|
self._end_date = end_date and pendulum.parse(end_date)
|
|
160
|
+
self._lookback_window = lookback_window
|
|
166
161
|
self.stream_params = {"dataTypeName": self.data_field}
|
|
167
162
|
|
|
168
163
|
def path(self, **kwargs) -> str:
|
|
@@ -268,6 +263,20 @@ class IterableExportStream(IterableStream, CheckpointMixin, ABC):
|
|
|
268
263
|
start_datetime = pendulum.parse(stream_state[self.cursor_field])
|
|
269
264
|
return start_datetime
|
|
270
265
|
|
|
266
|
+
def _get_effective_end_date(self) -> DateTime:
|
|
267
|
+
"""Compute the effective end date for the sync window.
|
|
268
|
+
|
|
269
|
+
When no explicit end_date is configured (production usage), the end
|
|
270
|
+
of the sync window is ``now() - lookback_window``. The
|
|
271
|
+
buffer accounts for Iterable Export API eventual consistency —
|
|
272
|
+
recently created events may not yet be indexed by the export
|
|
273
|
+
pipeline. Without this buffer the cursor would advance past those
|
|
274
|
+
events and they would be permanently lost.
|
|
275
|
+
"""
|
|
276
|
+
if self._end_date:
|
|
277
|
+
return self._end_date
|
|
278
|
+
return pendulum.now("UTC") - pendulum.Duration(minutes=self._lookback_window)
|
|
279
|
+
|
|
271
280
|
def stream_slices(
|
|
272
281
|
self,
|
|
273
282
|
sync_mode: SyncMode,
|
|
@@ -275,7 +284,7 @@ class IterableExportStream(IterableStream, CheckpointMixin, ABC):
|
|
|
275
284
|
stream_state: Mapping[str, Any] = None,
|
|
276
285
|
) -> Iterable[Optional[StreamSlice]]:
|
|
277
286
|
start_datetime = self.get_start_date(stream_state)
|
|
278
|
-
return [StreamSlice(start_datetime, self.
|
|
287
|
+
return [StreamSlice(start_datetime, self._get_effective_end_date())]
|
|
279
288
|
|
|
280
289
|
|
|
281
290
|
class IterableExportStreamRanged(IterableExportStream, ABC):
|
|
@@ -293,7 +302,7 @@ class IterableExportStreamRanged(IterableExportStream, ABC):
|
|
|
293
302
|
) -> Iterable[Optional[StreamSlice]]:
|
|
294
303
|
start_datetime = self.get_start_date(stream_state)
|
|
295
304
|
|
|
296
|
-
return RangeSliceGenerator(start_datetime, self.
|
|
305
|
+
return RangeSliceGenerator(start_datetime, self._get_effective_end_date())
|
|
297
306
|
|
|
298
307
|
|
|
299
308
|
class IterableExportStreamAdjustableRange(IterableExportStream, ABC):
|
|
@@ -326,7 +335,7 @@ class IterableExportStreamAdjustableRange(IterableExportStream, ABC):
|
|
|
326
335
|
stream_state: Mapping[str, Any] = None,
|
|
327
336
|
) -> Iterable[Optional[StreamSlice]]:
|
|
328
337
|
start_datetime = self.get_start_date(stream_state)
|
|
329
|
-
self._adjustable_generator = AdjustableSliceGenerator(start_datetime, self.
|
|
338
|
+
self._adjustable_generator = AdjustableSliceGenerator(start_datetime, self._get_effective_end_date())
|
|
330
339
|
return self._adjustable_generator
|
|
331
340
|
|
|
332
341
|
def read_records(
|
|
@@ -398,7 +407,7 @@ class CampaignsMetrics(IterableStream):
|
|
|
398
407
|
return params
|
|
399
408
|
|
|
400
409
|
def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
|
|
401
|
-
lists = Campaigns(authenticator=self._cred
|
|
410
|
+
lists = Campaigns(authenticator=self._cred)
|
|
402
411
|
campaign_ids = []
|
|
403
412
|
for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)):
|
|
404
413
|
campaign_ids.append(list_record["id"])
|
|
@@ -1,114 +0,0 @@
|
|
|
1
|
-
#
|
|
2
|
-
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
-
#
|
|
4
|
-
|
|
5
|
-
from typing import Any, List, Mapping
|
|
6
|
-
|
|
7
|
-
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
|
8
|
-
from airbyte_cdk.sources.streams import Stream
|
|
9
|
-
from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
|
|
10
|
-
|
|
11
|
-
from .streams import (
|
|
12
|
-
CampaignsMetrics,
|
|
13
|
-
CustomEvent,
|
|
14
|
-
EmailBounce,
|
|
15
|
-
EmailClick,
|
|
16
|
-
EmailComplaint,
|
|
17
|
-
EmailOpen,
|
|
18
|
-
EmailSend,
|
|
19
|
-
EmailSendSkip,
|
|
20
|
-
EmailSubscribe,
|
|
21
|
-
EmailUnsubscribe,
|
|
22
|
-
HostedUnsubscribeClick,
|
|
23
|
-
InAppClick,
|
|
24
|
-
InAppClose,
|
|
25
|
-
InAppDelete,
|
|
26
|
-
InAppDelivery,
|
|
27
|
-
InAppOpen,
|
|
28
|
-
InAppSend,
|
|
29
|
-
InAppSendSkip,
|
|
30
|
-
InboxMessageImpression,
|
|
31
|
-
InboxSession,
|
|
32
|
-
Purchase,
|
|
33
|
-
PushBounce,
|
|
34
|
-
PushOpen,
|
|
35
|
-
PushSend,
|
|
36
|
-
PushSendSkip,
|
|
37
|
-
PushUninstall,
|
|
38
|
-
SmsBounce,
|
|
39
|
-
SmsClick,
|
|
40
|
-
SmsReceived,
|
|
41
|
-
SmsSend,
|
|
42
|
-
SmsSendSkip,
|
|
43
|
-
SmsUsageInfo,
|
|
44
|
-
Templates,
|
|
45
|
-
WebPushClick,
|
|
46
|
-
WebPushSend,
|
|
47
|
-
WebPushSendSkip,
|
|
48
|
-
)
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
"""
|
|
52
|
-
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
|
|
53
|
-
source connector.
|
|
54
|
-
|
|
55
|
-
WARNING: Do not modify this file.
|
|
56
|
-
"""
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
# Declarative Source
|
|
60
|
-
class SourceIterable(YamlDeclarativeSource):
|
|
61
|
-
def __init__(self):
|
|
62
|
-
super().__init__(**{"path_to_yaml": "manifest.yaml"})
|
|
63
|
-
|
|
64
|
-
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
|
65
|
-
streams = super().streams(config=config)
|
|
66
|
-
|
|
67
|
-
authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
|
|
68
|
-
# end date is provided for integration tests only
|
|
69
|
-
start_date, end_date = config["start_date"], config.get("end_date")
|
|
70
|
-
date_range = {"start_date": start_date, "end_date": end_date}
|
|
71
|
-
region = config.get("region", "US")
|
|
72
|
-
|
|
73
|
-
# TODO: migrate streams below to low code as slicer logic will be migrated to generator based
|
|
74
|
-
streams.extend(
|
|
75
|
-
[
|
|
76
|
-
CampaignsMetrics(authenticator=authenticator, region=region, **date_range),
|
|
77
|
-
Templates(authenticator=authenticator, region=region, **date_range),
|
|
78
|
-
EmailBounce(authenticator=authenticator, region=region, **date_range),
|
|
79
|
-
EmailClick(authenticator=authenticator, region=region, **date_range),
|
|
80
|
-
EmailComplaint(authenticator=authenticator, region=region, **date_range),
|
|
81
|
-
EmailOpen(authenticator=authenticator, region=region, **date_range),
|
|
82
|
-
EmailSend(authenticator=authenticator, region=region, **date_range),
|
|
83
|
-
EmailSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
84
|
-
EmailSubscribe(authenticator=authenticator, region=region, **date_range),
|
|
85
|
-
EmailUnsubscribe(authenticator=authenticator, region=region, **date_range),
|
|
86
|
-
PushSend(authenticator=authenticator, region=region, **date_range),
|
|
87
|
-
PushSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
88
|
-
PushOpen(authenticator=authenticator, region=region, **date_range),
|
|
89
|
-
PushUninstall(authenticator=authenticator, region=region, **date_range),
|
|
90
|
-
PushBounce(authenticator=authenticator, region=region, **date_range),
|
|
91
|
-
WebPushSend(authenticator=authenticator, region=region, **date_range),
|
|
92
|
-
WebPushClick(authenticator=authenticator, region=region, **date_range),
|
|
93
|
-
WebPushSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
94
|
-
InAppSend(authenticator=authenticator, region=region, **date_range),
|
|
95
|
-
InAppOpen(authenticator=authenticator, region=region, **date_range),
|
|
96
|
-
InAppClick(authenticator=authenticator, region=region, **date_range),
|
|
97
|
-
InAppClose(authenticator=authenticator, region=region, **date_range),
|
|
98
|
-
InAppDelete(authenticator=authenticator, region=region, **date_range),
|
|
99
|
-
InAppDelivery(authenticator=authenticator, region=region, **date_range),
|
|
100
|
-
InAppSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
101
|
-
InboxSession(authenticator=authenticator, region=region, **date_range),
|
|
102
|
-
InboxMessageImpression(authenticator=authenticator, region=region, **date_range),
|
|
103
|
-
SmsSend(authenticator=authenticator, region=region, **date_range),
|
|
104
|
-
SmsBounce(authenticator=authenticator, region=region, **date_range),
|
|
105
|
-
SmsClick(authenticator=authenticator, region=region, **date_range),
|
|
106
|
-
SmsReceived(authenticator=authenticator, region=region, **date_range),
|
|
107
|
-
SmsSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
108
|
-
SmsUsageInfo(authenticator=authenticator, region=region, **date_range),
|
|
109
|
-
Purchase(authenticator=authenticator, region=region, **date_range),
|
|
110
|
-
CustomEvent(authenticator=authenticator, region=region, **date_range),
|
|
111
|
-
HostedUnsubscribeClick(authenticator=authenticator, region=region, **date_range),
|
|
112
|
-
]
|
|
113
|
-
)
|
|
114
|
-
return streams
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|