airbyte-source-iterable 0.7.1__tar.gz → 1.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- airbyte_source_iterable-1.0.0/.airbyte-pypi-readme.md +7 -0
- airbyte_source_iterable-1.0.0/PKG-INFO +29 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/pyproject.toml +2 -2
- airbyte_source_iterable-1.0.0/source_iterable/components.py +126 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/manifest.yaml +26 -14
- airbyte_source_iterable-1.0.0/source_iterable/schemas/users.json +159 -0
- airbyte_source_iterable-1.0.0/source_iterable/source.py +117 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/spec.json +9 -1
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/streams.py +9 -3
- airbyte_source_iterable-0.7.1/PKG-INFO +0 -126
- airbyte_source_iterable-0.7.1/README.md +0 -104
- airbyte_source_iterable-0.7.1/source_iterable/components.py +0 -23
- airbyte_source_iterable-0.7.1/source_iterable/schemas/users.json +0 -458
- airbyte_source_iterable-0.7.1/source_iterable/source.py +0 -116
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/__init__.py +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/run.py +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/campaigns.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/campaigns_metrics.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/channels.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_bounce.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_click.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_complaint.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_open.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_send.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_send_skip.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_subscribe.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/email_unsubscribe.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/events.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/list_users.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/lists.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/message_types.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/metadata.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/schemas/templates.json +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/slice_generators.py +0 -0
- {airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/utils.py +0 -0
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
# Iterable
|
|
2
|
+
|
|
3
|
+
This package contains the Airbyte Iterable connector.
|
|
4
|
+
|
|
5
|
+
Airbyte is open-source data movement for ELT pipelines and AI agents. It moves data from APIs, databases, and files to warehouses, lakes, and AI applications.
|
|
6
|
+
|
|
7
|
+
Airbyte provides a catalog of 600+ connectors for APIs, databases, data warehouses, data lakes, and AI applications. Use this package with [PyAirbyte](https://docs.airbyte.com/developers/pyairbyte) to run the connector from Python.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
Metadata-Version: 2.1
|
|
2
|
+
Name: airbyte-source-iterable
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Source implementation for Iterable.
|
|
5
|
+
Home-page: https://airbyte.com
|
|
6
|
+
License: ELv2
|
|
7
|
+
Author: Airbyte
|
|
8
|
+
Author-email: contact@airbyte.io
|
|
9
|
+
Requires-Python: >=3.9,<3.12
|
|
10
|
+
Classifier: License :: Other/Proprietary License
|
|
11
|
+
Classifier: Programming Language :: Python :: 3
|
|
12
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
15
|
+
Requires-Dist: airbyte-cdk (>=3,<4)
|
|
16
|
+
Requires-Dist: pendulum (==2.1.2)
|
|
17
|
+
Requires-Dist: python-dateutil (==2.8.2)
|
|
18
|
+
Project-URL: Documentation, https://docs.airbyte.com/integrations/sources/iterable
|
|
19
|
+
Project-URL: Repository, https://github.com/airbytehq/airbyte
|
|
20
|
+
Description-Content-Type: text/markdown
|
|
21
|
+
|
|
22
|
+
# Iterable
|
|
23
|
+
|
|
24
|
+
This package contains the Airbyte Iterable connector.
|
|
25
|
+
|
|
26
|
+
Airbyte is open-source data movement for ELT pipelines and AI agents. It moves data from APIs, databases, and files to warehouses, lakes, and AI applications.
|
|
27
|
+
|
|
28
|
+
Airbyte provides a catalog of 600+ connectors for APIs, databases, data warehouses, data lakes, and AI applications. Use this package with [PyAirbyte](https://docs.airbyte.com/developers/pyairbyte) to run the connector from Python.
|
|
29
|
+
|
|
@@ -3,12 +3,12 @@ requires = [ "poetry-core>=1.0.0",]
|
|
|
3
3
|
build-backend = "poetry.core.masonry.api"
|
|
4
4
|
|
|
5
5
|
[tool.poetry]
|
|
6
|
-
version = "0.
|
|
6
|
+
version = "1.0.0"
|
|
7
7
|
name = "airbyte-source-iterable"
|
|
8
8
|
description = "Source implementation for Iterable."
|
|
9
9
|
authors = [ "Airbyte <contact@airbyte.io>",]
|
|
10
10
|
license = "ELv2"
|
|
11
|
-
readme = "
|
|
11
|
+
readme = ".airbyte-pypi-readme.md"
|
|
12
12
|
documentation = "https://docs.airbyte.com/integrations/sources/iterable"
|
|
13
13
|
homepage = "https://airbyte.com"
|
|
14
14
|
repository = "https://github.com/airbytehq/airbyte"
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import datetime
|
|
7
|
+
from typing import Any, Iterable, Mapping
|
|
8
|
+
|
|
9
|
+
import requests
|
|
10
|
+
|
|
11
|
+
from airbyte_cdk.sources.declarative.extractors.dpath_extractor import DpathExtractor
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
@dataclass
|
|
15
|
+
class EventsRecordExtractor(DpathExtractor):
|
|
16
|
+
common_fields = ("itblInternal", "_type", "createdAt", "email")
|
|
17
|
+
|
|
18
|
+
def extract_records(self, response: requests.Response) -> Iterable[Mapping[str, Any]]:
|
|
19
|
+
jsonl_records = super().extract_records(response=response)
|
|
20
|
+
for record_dict in jsonl_records:
|
|
21
|
+
record_dict_common_fields = {}
|
|
22
|
+
for field in self.common_fields:
|
|
23
|
+
record_dict_common_fields[field] = record_dict.pop(field, None)
|
|
24
|
+
yield {**record_dict_common_fields, "data": record_dict}
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass
|
|
28
|
+
class UsersRecordExtractor(DpathExtractor):
|
|
29
|
+
"""Extracts user records, keeping standard Iterable-managed fields at the top level
|
|
30
|
+
and placing all custom/tenant-specific dataFields into a generic `data` object.
|
|
31
|
+
|
|
32
|
+
Standard fields are those documented at:
|
|
33
|
+
https://support.iterable.com/hc/en-us/articles/217744303-User-Profile-Fields-Used-by-Iterable
|
|
34
|
+
|
|
35
|
+
The Iterable export API flattens nested objects into dotted keys (e.g.
|
|
36
|
+
`itblInternal.emailDomain`), so Iterable-internal fields are listed in their flat
|
|
37
|
+
dotted form. Keys not listed here - including Iterable-internal dotted keys that
|
|
38
|
+
are not declared in the stream schema - are routed into `data` so destinations
|
|
39
|
+
that materialize only declared fields never drop them.
|
|
40
|
+
"""
|
|
41
|
+
|
|
42
|
+
# Fields documented by Iterable as managed or used for sending messages.
|
|
43
|
+
# Reference: https://support.iterable.com/hc/en-us/articles/217744303
|
|
44
|
+
STANDARD_FIELDS = frozenset(
|
|
45
|
+
{
|
|
46
|
+
# Unique identifiers
|
|
47
|
+
"email",
|
|
48
|
+
"userId",
|
|
49
|
+
# Managed by Iterable
|
|
50
|
+
"emailListIds",
|
|
51
|
+
"itblUserId",
|
|
52
|
+
"knownLitigatorFilter",
|
|
53
|
+
"profileUpdatedAt",
|
|
54
|
+
"receivedSMSDisclaimer",
|
|
55
|
+
"signupDate",
|
|
56
|
+
"signupSource",
|
|
57
|
+
"subscribedMessageTypeIds",
|
|
58
|
+
"unsubscribedChannelIds",
|
|
59
|
+
"unsubscribedMessageTypeIds",
|
|
60
|
+
"userListIds",
|
|
61
|
+
# Geolocation fields populated by Iterable from the user's IP
|
|
62
|
+
"city",
|
|
63
|
+
"region",
|
|
64
|
+
# Fields used for sending messages
|
|
65
|
+
"country",
|
|
66
|
+
"devices",
|
|
67
|
+
"ip",
|
|
68
|
+
"locale",
|
|
69
|
+
"phoneNumber",
|
|
70
|
+
"profile",
|
|
71
|
+
"timeZone",
|
|
72
|
+
"whatsAppPhoneNumber",
|
|
73
|
+
# Iterable-internal fields (the export returns them as flat dotted keys)
|
|
74
|
+
"itblInternal.emailDomain",
|
|
75
|
+
"itblInternal.phoneCountry",
|
|
76
|
+
"itblInternal.phoneType",
|
|
77
|
+
"itblInternal.documentCreatedAt",
|
|
78
|
+
"itblInternal.documentUpdatedAt",
|
|
79
|
+
"itblInternal.isUnknownUser",
|
|
80
|
+
"itblDS.brandAffinityLabel",
|
|
81
|
+
"itblDS.predictiveGoals",
|
|
82
|
+
}
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
# The export API emits these as space-separated timestamps ("2021-04-14 16:52:31 +00:00")
|
|
86
|
+
# while the schema declares format: date-time (RFC3339); normalize so typed destinations
|
|
87
|
+
# do not null the values into _airbyte_meta.changes.
|
|
88
|
+
TIMESTAMP_FIELDS = (
|
|
89
|
+
"signupDate",
|
|
90
|
+
"profileUpdatedAt",
|
|
91
|
+
"itblInternal.documentCreatedAt",
|
|
92
|
+
"itblInternal.documentUpdatedAt",
|
|
93
|
+
)
|
|
94
|
+
_TIMESTAMP_FORMATS = (
|
|
95
|
+
"%Y-%m-%d %H:%M:%S %z",
|
|
96
|
+
"%Y-%m-%dT%H:%M:%S%z",
|
|
97
|
+
"%Y-%m-%d %H:%M:%S.%f %z",
|
|
98
|
+
"%Y-%m-%dT%H:%M:%S.%f%z",
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
@classmethod
|
|
102
|
+
def _normalize_timestamp(cls, value: Any) -> Any:
|
|
103
|
+
if not isinstance(value, str):
|
|
104
|
+
return value
|
|
105
|
+
for fmt in cls._TIMESTAMP_FORMATS:
|
|
106
|
+
try:
|
|
107
|
+
return datetime.strptime(value, fmt).isoformat()
|
|
108
|
+
except ValueError:
|
|
109
|
+
continue
|
|
110
|
+
return value
|
|
111
|
+
|
|
112
|
+
def extract_records(self, response: requests.Response) -> Iterable[Mapping[str, Any]]:
|
|
113
|
+
jsonl_records = super().extract_records(response=response)
|
|
114
|
+
for record_dict in jsonl_records:
|
|
115
|
+
standard: dict[str, Any] = {}
|
|
116
|
+
data: dict[str, Any] = {}
|
|
117
|
+
for key, value in record_dict.items():
|
|
118
|
+
if key in self.STANDARD_FIELDS:
|
|
119
|
+
standard[key] = value
|
|
120
|
+
else:
|
|
121
|
+
data[key] = value
|
|
122
|
+
for field in self.TIMESTAMP_FIELDS:
|
|
123
|
+
if field in standard:
|
|
124
|
+
standard[field] = self._normalize_timestamp(standard[field])
|
|
125
|
+
standard["data"] = data
|
|
126
|
+
yield standard
|
{airbyte_source_iterable-0.7.1 → airbyte_source_iterable-1.0.0}/source_iterable/manifest.yaml
RENAMED
|
@@ -21,12 +21,22 @@ spec:
|
|
|
21
21
|
type: "string"
|
|
22
22
|
title: "Start Date"
|
|
23
23
|
description: >-
|
|
24
|
-
The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.
|
|
24
|
+
The date from which you'd like to replicate data for Iterable, in the format YYYY-MM-DDT00:00:00Z.
|
|
25
25
|
All data generated after this date will be replicated.
|
|
26
26
|
examples: ["2021-04-01T00:00:00Z"]
|
|
27
27
|
pattern: ^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$
|
|
28
28
|
order: 1
|
|
29
29
|
format: "date-time"
|
|
30
|
+
region:
|
|
31
|
+
type: "string"
|
|
32
|
+
title: "Region"
|
|
33
|
+
description: >-
|
|
34
|
+
The region where your Iterable account is hosted. Select 'EU' if your account is on the European data center.
|
|
35
|
+
enum:
|
|
36
|
+
- US
|
|
37
|
+
- EU
|
|
38
|
+
default: "US"
|
|
39
|
+
order: 2
|
|
30
40
|
lookback_window:
|
|
31
41
|
type: "integer"
|
|
32
42
|
title: "Lookback Window (Minutes)"
|
|
@@ -38,7 +48,7 @@ spec:
|
|
|
38
48
|
default: 5
|
|
39
49
|
minimum: 0
|
|
40
50
|
examples: [5, 10, 15]
|
|
41
|
-
order:
|
|
51
|
+
order: 3
|
|
42
52
|
type: DeclarativeSource
|
|
43
53
|
check:
|
|
44
54
|
type: CheckStream
|
|
@@ -54,7 +64,7 @@ streams:
|
|
|
54
64
|
requester:
|
|
55
65
|
path: lists
|
|
56
66
|
type: HttpRequester
|
|
57
|
-
url_base: https://api.iterable.com/api/
|
|
67
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
58
68
|
http_method: GET
|
|
59
69
|
authenticator:
|
|
60
70
|
type: ApiKeyAuthenticator
|
|
@@ -86,7 +96,7 @@ streams:
|
|
|
86
96
|
requester:
|
|
87
97
|
path: lists/getUsers
|
|
88
98
|
type: HttpRequester
|
|
89
|
-
url_base: https://api.iterable.com/api/
|
|
99
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
90
100
|
http_method: GET
|
|
91
101
|
authenticator:
|
|
92
102
|
type: ApiKeyAuthenticator
|
|
@@ -122,7 +132,7 @@ streams:
|
|
|
122
132
|
type: SimpleRetriever
|
|
123
133
|
requester:
|
|
124
134
|
type: HttpRequester
|
|
125
|
-
url_base: https://api.iterable.com/api/
|
|
135
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
126
136
|
path: lists
|
|
127
137
|
http_method: GET
|
|
128
138
|
request_parameters: {}
|
|
@@ -167,7 +177,7 @@ streams:
|
|
|
167
177
|
requester:
|
|
168
178
|
path: campaigns
|
|
169
179
|
type: HttpRequester
|
|
170
|
-
url_base: https://api.iterable.com/api/
|
|
180
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
171
181
|
http_method: GET
|
|
172
182
|
authenticator:
|
|
173
183
|
type: ApiKeyAuthenticator
|
|
@@ -197,7 +207,7 @@ streams:
|
|
|
197
207
|
requester:
|
|
198
208
|
path: channels
|
|
199
209
|
type: HttpRequester
|
|
200
|
-
url_base: https://api.iterable.com/api/
|
|
210
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
201
211
|
http_method: GET
|
|
202
212
|
authenticator:
|
|
203
213
|
type: ApiKeyAuthenticator
|
|
@@ -227,7 +237,7 @@ streams:
|
|
|
227
237
|
requester:
|
|
228
238
|
path: messageTypes
|
|
229
239
|
type: HttpRequester
|
|
230
|
-
url_base: https://api.iterable.com/api/
|
|
240
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
231
241
|
http_method: GET
|
|
232
242
|
authenticator:
|
|
233
243
|
type: ApiKeyAuthenticator
|
|
@@ -257,7 +267,7 @@ streams:
|
|
|
257
267
|
requester:
|
|
258
268
|
path: metadata
|
|
259
269
|
type: HttpRequester
|
|
260
|
-
url_base: https://api.iterable.com/api/
|
|
270
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
261
271
|
http_method: GET
|
|
262
272
|
authenticator:
|
|
263
273
|
type: ApiKeyAuthenticator
|
|
@@ -288,7 +298,7 @@ streams:
|
|
|
288
298
|
requester:
|
|
289
299
|
path: export/data.json
|
|
290
300
|
type: HttpRequester
|
|
291
|
-
url_base: https://api.iterable.com/api/
|
|
301
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
292
302
|
http_method: GET
|
|
293
303
|
authenticator:
|
|
294
304
|
type: ApiKeyAuthenticator
|
|
@@ -305,7 +315,7 @@ streams:
|
|
|
305
315
|
record_selector:
|
|
306
316
|
type: RecordSelector
|
|
307
317
|
extractor:
|
|
308
|
-
|
|
318
|
+
class_name: source_iterable.components.UsersRecordExtractor
|
|
309
319
|
field_path: []
|
|
310
320
|
partition_router: []
|
|
311
321
|
primary_key: []
|
|
@@ -334,6 +344,8 @@ streams:
|
|
|
334
344
|
cursor_datetime_formats:
|
|
335
345
|
- "%Y-%m-%d %H:%M:%S %z"
|
|
336
346
|
- "%Y-%m-%dT%H:%M:%S%z"
|
|
347
|
+
- "%Y-%m-%d %H:%M:%S.%f %z"
|
|
348
|
+
- "%Y-%m-%dT%H:%M:%S.%f%z"
|
|
337
349
|
- name: events
|
|
338
350
|
primary_key: []
|
|
339
351
|
retriever:
|
|
@@ -342,7 +354,7 @@ streams:
|
|
|
342
354
|
type: JsonlDecoder
|
|
343
355
|
requester:
|
|
344
356
|
type: HttpRequester
|
|
345
|
-
url_base: https://api.iterable.com/api/
|
|
357
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
346
358
|
path: export/userEvents
|
|
347
359
|
http_method: GET
|
|
348
360
|
request_parameters:
|
|
@@ -385,7 +397,7 @@ streams:
|
|
|
385
397
|
requester:
|
|
386
398
|
path: lists/getUsers
|
|
387
399
|
type: HttpRequester
|
|
388
|
-
url_base: https://api.iterable.com/api/
|
|
400
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
389
401
|
http_method: GET
|
|
390
402
|
authenticator:
|
|
391
403
|
type: ApiKeyAuthenticator
|
|
@@ -421,7 +433,7 @@ streams:
|
|
|
421
433
|
type: SimpleRetriever
|
|
422
434
|
requester:
|
|
423
435
|
type: HttpRequester
|
|
424
|
-
url_base: https://api.iterable.com/api/
|
|
436
|
+
url_base: "{{ 'https://api.eu.iterable.com/api/' if config.get('region', 'US') == 'EU' else 'https://api.iterable.com/api/' }}"
|
|
425
437
|
path: lists
|
|
426
438
|
http_method: GET
|
|
427
439
|
request_parameters: {}
|
|
@@ -0,0 +1,159 @@
|
|
|
1
|
+
{
|
|
2
|
+
"properties": {
|
|
3
|
+
"email": {
|
|
4
|
+
"description": "User's email address (unique identifier).",
|
|
5
|
+
"type": ["null", "string"]
|
|
6
|
+
},
|
|
7
|
+
"userId": {
|
|
8
|
+
"description": "User's unique identifier assigned by the tenant.",
|
|
9
|
+
"type": ["null", "string"]
|
|
10
|
+
},
|
|
11
|
+
"emailListIds": {
|
|
12
|
+
"description": "List of static list IDs the user is a member of (email-based projects).",
|
|
13
|
+
"type": ["null", "array"],
|
|
14
|
+
"items": {
|
|
15
|
+
"type": ["null", "integer"]
|
|
16
|
+
}
|
|
17
|
+
},
|
|
18
|
+
"itblUserId": {
|
|
19
|
+
"description": "Iterable's internal unique user identifier.",
|
|
20
|
+
"type": ["null", "string"]
|
|
21
|
+
},
|
|
22
|
+
"knownLitigatorFilter": {
|
|
23
|
+
"description": "Known litigator confidence level (LT1, LT2, or LT3).",
|
|
24
|
+
"type": ["null", "string"]
|
|
25
|
+
},
|
|
26
|
+
"profileUpdatedAt": {
|
|
27
|
+
"description": "Date and time when the profile was last updated.",
|
|
28
|
+
"type": ["null", "string"],
|
|
29
|
+
"format": "date-time",
|
|
30
|
+
"airbyte_type": "timestamp_with_timezone"
|
|
31
|
+
},
|
|
32
|
+
"receivedSMSDisclaimer": {
|
|
33
|
+
"description": "Whether the user has received an SMS with opt-out instructions.",
|
|
34
|
+
"type": ["null", "boolean"]
|
|
35
|
+
},
|
|
36
|
+
"signupDate": {
|
|
37
|
+
"description": "Date and time when the user profile was created.",
|
|
38
|
+
"type": ["null", "string"],
|
|
39
|
+
"format": "date-time",
|
|
40
|
+
"airbyte_type": "timestamp_with_timezone"
|
|
41
|
+
},
|
|
42
|
+
"signupSource": {
|
|
43
|
+
"description": "How the contact was created in Iterable (e.g. API, IMPORT).",
|
|
44
|
+
"type": ["null", "string"]
|
|
45
|
+
},
|
|
46
|
+
"subscribedMessageTypeIds": {
|
|
47
|
+
"description": "List of opt-in message type IDs the user has subscribed to.",
|
|
48
|
+
"type": ["null", "array"],
|
|
49
|
+
"items": {
|
|
50
|
+
"type": ["null", "integer"]
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
"unsubscribedChannelIds": {
|
|
54
|
+
"description": "List of channel IDs the user has unsubscribed from.",
|
|
55
|
+
"type": ["null", "array"],
|
|
56
|
+
"items": {
|
|
57
|
+
"type": ["null", "integer"]
|
|
58
|
+
}
|
|
59
|
+
},
|
|
60
|
+
"unsubscribedMessageTypeIds": {
|
|
61
|
+
"description": "List of opt-out message type IDs the user has unsubscribed from.",
|
|
62
|
+
"type": ["null", "array"],
|
|
63
|
+
"items": {
|
|
64
|
+
"type": ["null", "integer"]
|
|
65
|
+
}
|
|
66
|
+
},
|
|
67
|
+
"userListIds": {
|
|
68
|
+
"description": "List of static list IDs the user is a member of (userId/hybrid projects).",
|
|
69
|
+
"type": ["null", "array"],
|
|
70
|
+
"items": {
|
|
71
|
+
"type": ["null", "integer"]
|
|
72
|
+
}
|
|
73
|
+
},
|
|
74
|
+
"country": {
|
|
75
|
+
"description": "User's country.",
|
|
76
|
+
"type": ["null", "string"]
|
|
77
|
+
},
|
|
78
|
+
"city": {
|
|
79
|
+
"description": "User's city, populated by Iterable from the user's IP address when the Overwrite Location Fields project setting is on.",
|
|
80
|
+
"type": ["null", "string"]
|
|
81
|
+
},
|
|
82
|
+
"region": {
|
|
83
|
+
"description": "User's region, populated by Iterable from the user's IP address when the Overwrite Location Fields project setting is on.",
|
|
84
|
+
"type": ["null", "string"]
|
|
85
|
+
},
|
|
86
|
+
"devices": {
|
|
87
|
+
"description": "Devices registered for push notifications.",
|
|
88
|
+
"type": ["null", "array"],
|
|
89
|
+
"items": {
|
|
90
|
+
"type": ["null", "object"]
|
|
91
|
+
}
|
|
92
|
+
},
|
|
93
|
+
"ip": {
|
|
94
|
+
"description": "User's IP address.",
|
|
95
|
+
"type": ["null", "string"]
|
|
96
|
+
},
|
|
97
|
+
"locale": {
|
|
98
|
+
"description": "User's locale.",
|
|
99
|
+
"type": ["null", "string"]
|
|
100
|
+
},
|
|
101
|
+
"phoneNumber": {
|
|
102
|
+
"description": "User's phone number used for SMS.",
|
|
103
|
+
"type": ["null", "string"]
|
|
104
|
+
},
|
|
105
|
+
"whatsAppPhoneNumber": {
|
|
106
|
+
"description": "User's phone number used as the recipient identifier for WhatsApp campaigns.",
|
|
107
|
+
"type": ["null", "string"]
|
|
108
|
+
},
|
|
109
|
+
"profile": {
|
|
110
|
+
"description": "Profile data used for message personalization.",
|
|
111
|
+
"type": ["null", "object"]
|
|
112
|
+
},
|
|
113
|
+
"timeZone": {
|
|
114
|
+
"description": "User's time zone.",
|
|
115
|
+
"type": ["null", "string"]
|
|
116
|
+
},
|
|
117
|
+
"itblInternal.emailDomain": {
|
|
118
|
+
"description": "Domain of the user's email address (managed by Iterable).",
|
|
119
|
+
"type": ["null", "string"]
|
|
120
|
+
},
|
|
121
|
+
"itblInternal.phoneCountry": {
|
|
122
|
+
"description": "ISO-3166 country code for the user's phone number.",
|
|
123
|
+
"type": ["null", "string"]
|
|
124
|
+
},
|
|
125
|
+
"itblInternal.phoneType": {
|
|
126
|
+
"description": "Type of phone number (MOBILE, FIXED_LINE, VOIP, etc.).",
|
|
127
|
+
"type": ["null", "string"]
|
|
128
|
+
},
|
|
129
|
+
"itblInternal.documentCreatedAt": {
|
|
130
|
+
"description": "Timestamp when the user document was created internally.",
|
|
131
|
+
"type": ["null", "string"],
|
|
132
|
+
"format": "date-time",
|
|
133
|
+
"airbyte_type": "timestamp_with_timezone"
|
|
134
|
+
},
|
|
135
|
+
"itblInternal.documentUpdatedAt": {
|
|
136
|
+
"description": "Timestamp when the user document was last updated internally.",
|
|
137
|
+
"type": ["null", "string"],
|
|
138
|
+
"format": "date-time",
|
|
139
|
+
"airbyte_type": "timestamp_with_timezone"
|
|
140
|
+
},
|
|
141
|
+
"itblInternal.isUnknownUser": {
|
|
142
|
+
"description": "Whether the user is flagged as unknown by Iterable.",
|
|
143
|
+
"type": ["null", "boolean"]
|
|
144
|
+
},
|
|
145
|
+
"itblDS.brandAffinityLabel": {
|
|
146
|
+
"description": "Contact's engagement level as determined by Brand Affinity.",
|
|
147
|
+
"type": ["null", "string"]
|
|
148
|
+
},
|
|
149
|
+
"itblDS.predictiveGoals": {
|
|
150
|
+
"description": "User conversion scores for Predictive Goals.",
|
|
151
|
+
"type": ["null", "object"]
|
|
152
|
+
},
|
|
153
|
+
"data": {
|
|
154
|
+
"description": "Custom data fields specific to the tenant. Contains all non-standard user profile fields.",
|
|
155
|
+
"type": ["null", "object"]
|
|
156
|
+
}
|
|
157
|
+
},
|
|
158
|
+
"type": ["null", "object"]
|
|
159
|
+
}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
#
|
|
2
|
+
# Copyright (c) 2023 Airbyte, Inc., all rights reserved.
|
|
3
|
+
#
|
|
4
|
+
|
|
5
|
+
from typing import Any, List, Mapping
|
|
6
|
+
|
|
7
|
+
from airbyte_cdk.sources.declarative.yaml_declarative_source import YamlDeclarativeSource
|
|
8
|
+
from airbyte_cdk.sources.streams import Stream
|
|
9
|
+
from airbyte_cdk.sources.streams.http.requests_native_auth import TokenAuthenticator
|
|
10
|
+
|
|
11
|
+
from .streams import (
|
|
12
|
+
CampaignsMetrics,
|
|
13
|
+
CustomEvent,
|
|
14
|
+
EmailBounce,
|
|
15
|
+
EmailClick,
|
|
16
|
+
EmailComplaint,
|
|
17
|
+
EmailOpen,
|
|
18
|
+
EmailSend,
|
|
19
|
+
EmailSendSkip,
|
|
20
|
+
EmailSubscribe,
|
|
21
|
+
EmailUnsubscribe,
|
|
22
|
+
HostedUnsubscribeClick,
|
|
23
|
+
InAppClick,
|
|
24
|
+
InAppClose,
|
|
25
|
+
InAppDelete,
|
|
26
|
+
InAppDelivery,
|
|
27
|
+
InAppOpen,
|
|
28
|
+
InAppSend,
|
|
29
|
+
InAppSendSkip,
|
|
30
|
+
InboxMessageImpression,
|
|
31
|
+
InboxSession,
|
|
32
|
+
Purchase,
|
|
33
|
+
PushBounce,
|
|
34
|
+
PushOpen,
|
|
35
|
+
PushSend,
|
|
36
|
+
PushSendSkip,
|
|
37
|
+
PushUninstall,
|
|
38
|
+
SmsBounce,
|
|
39
|
+
SmsClick,
|
|
40
|
+
SmsReceived,
|
|
41
|
+
SmsSend,
|
|
42
|
+
SmsSendSkip,
|
|
43
|
+
SmsUsageInfo,
|
|
44
|
+
Templates,
|
|
45
|
+
WebPushClick,
|
|
46
|
+
WebPushSend,
|
|
47
|
+
WebPushSendSkip,
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
"""
|
|
52
|
+
This file provides the necessary constructs to interpret a provided declarative YAML configuration file into
|
|
53
|
+
source connector.
|
|
54
|
+
|
|
55
|
+
WARNING: Do not modify this file.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
# Declarative Source
|
|
60
|
+
class SourceIterable(YamlDeclarativeSource):
|
|
61
|
+
def __init__(self):
|
|
62
|
+
super().__init__(**{"path_to_yaml": "manifest.yaml"})
|
|
63
|
+
|
|
64
|
+
def streams(self, config: Mapping[str, Any]) -> List[Stream]:
|
|
65
|
+
streams = super().streams(config=config)
|
|
66
|
+
|
|
67
|
+
authenticator = TokenAuthenticator(token=config["api_key"], auth_header="Api-Key", auth_method="")
|
|
68
|
+
# end date is provided for integration tests only
|
|
69
|
+
start_date, end_date = config["start_date"], config.get("end_date")
|
|
70
|
+
lookback_window = config.get("lookback_window", 5)
|
|
71
|
+
date_range = {"start_date": start_date, "end_date": end_date, "lookback_window": lookback_window}
|
|
72
|
+
# CampaignsMetrics extends IterableStream (not IterableExportStream) and does not accept lookback_window
|
|
73
|
+
campaigns_date_range = {"start_date": start_date, "end_date": end_date}
|
|
74
|
+
region = config.get("region", "US")
|
|
75
|
+
|
|
76
|
+
# TODO: migrate streams below to low code as slicer logic will be migrated to generator based
|
|
77
|
+
streams.extend(
|
|
78
|
+
[
|
|
79
|
+
CampaignsMetrics(authenticator=authenticator, region=region, **campaigns_date_range),
|
|
80
|
+
Templates(authenticator=authenticator, region=region, **date_range),
|
|
81
|
+
EmailBounce(authenticator=authenticator, region=region, **date_range),
|
|
82
|
+
EmailClick(authenticator=authenticator, region=region, **date_range),
|
|
83
|
+
EmailComplaint(authenticator=authenticator, region=region, **date_range),
|
|
84
|
+
EmailOpen(authenticator=authenticator, region=region, **date_range),
|
|
85
|
+
EmailSend(authenticator=authenticator, region=region, **date_range),
|
|
86
|
+
EmailSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
87
|
+
EmailSubscribe(authenticator=authenticator, region=region, **date_range),
|
|
88
|
+
EmailUnsubscribe(authenticator=authenticator, region=region, **date_range),
|
|
89
|
+
PushSend(authenticator=authenticator, region=region, **date_range),
|
|
90
|
+
PushSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
91
|
+
PushOpen(authenticator=authenticator, region=region, **date_range),
|
|
92
|
+
PushUninstall(authenticator=authenticator, region=region, **date_range),
|
|
93
|
+
PushBounce(authenticator=authenticator, region=region, **date_range),
|
|
94
|
+
WebPushSend(authenticator=authenticator, region=region, **date_range),
|
|
95
|
+
WebPushClick(authenticator=authenticator, region=region, **date_range),
|
|
96
|
+
WebPushSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
97
|
+
InAppSend(authenticator=authenticator, region=region, **date_range),
|
|
98
|
+
InAppOpen(authenticator=authenticator, region=region, **date_range),
|
|
99
|
+
InAppClick(authenticator=authenticator, region=region, **date_range),
|
|
100
|
+
InAppClose(authenticator=authenticator, region=region, **date_range),
|
|
101
|
+
InAppDelete(authenticator=authenticator, region=region, **date_range),
|
|
102
|
+
InAppDelivery(authenticator=authenticator, region=region, **date_range),
|
|
103
|
+
InAppSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
104
|
+
InboxSession(authenticator=authenticator, region=region, **date_range),
|
|
105
|
+
InboxMessageImpression(authenticator=authenticator, region=region, **date_range),
|
|
106
|
+
SmsSend(authenticator=authenticator, region=region, **date_range),
|
|
107
|
+
SmsBounce(authenticator=authenticator, region=region, **date_range),
|
|
108
|
+
SmsClick(authenticator=authenticator, region=region, **date_range),
|
|
109
|
+
SmsReceived(authenticator=authenticator, region=region, **date_range),
|
|
110
|
+
SmsSendSkip(authenticator=authenticator, region=region, **date_range),
|
|
111
|
+
SmsUsageInfo(authenticator=authenticator, region=region, **date_range),
|
|
112
|
+
Purchase(authenticator=authenticator, region=region, **date_range),
|
|
113
|
+
CustomEvent(authenticator=authenticator, region=region, **date_range),
|
|
114
|
+
HostedUnsubscribeClick(authenticator=authenticator, region=region, **date_range),
|
|
115
|
+
]
|
|
116
|
+
)
|
|
117
|
+
return streams
|
|
@@ -22,6 +22,14 @@
|
|
|
22
22
|
"order": 1,
|
|
23
23
|
"format": "date-time"
|
|
24
24
|
},
|
|
25
|
+
"region": {
|
|
26
|
+
"type": "string",
|
|
27
|
+
"title": "Region",
|
|
28
|
+
"description": "The region where your Iterable account is hosted. Select 'EU' if your account is on the European data center.",
|
|
29
|
+
"enum": ["US", "EU"],
|
|
30
|
+
"default": "US",
|
|
31
|
+
"order": 2
|
|
32
|
+
},
|
|
25
33
|
"lookback_window": {
|
|
26
34
|
"type": "integer",
|
|
27
35
|
"title": "Lookback Window (Minutes)",
|
|
@@ -29,7 +37,7 @@
|
|
|
29
37
|
"default": 5,
|
|
30
38
|
"minimum": 0,
|
|
31
39
|
"examples": [5, 10, 15],
|
|
32
|
-
"order":
|
|
40
|
+
"order": 3
|
|
33
41
|
}
|
|
34
42
|
}
|
|
35
43
|
}
|
|
@@ -33,14 +33,20 @@ class IterableStream(HttpStream, ABC):
|
|
|
33
33
|
# to prevent 429 error on other streams
|
|
34
34
|
ignore_further_slices = False
|
|
35
35
|
|
|
36
|
-
url_base = "https://api.iterable.com/api/"
|
|
37
36
|
primary_key = "id"
|
|
38
37
|
|
|
39
|
-
def __init__(self, authenticator):
|
|
38
|
+
def __init__(self, authenticator, region: str = "US"):
|
|
40
39
|
self._cred = authenticator
|
|
41
40
|
self._slice_retry = 0
|
|
41
|
+
self._region = region
|
|
42
42
|
super().__init__(authenticator)
|
|
43
43
|
|
|
44
|
+
@property
|
|
45
|
+
def url_base(self) -> str:
|
|
46
|
+
if self._region == "EU":
|
|
47
|
+
return "https://api.eu.iterable.com/api/"
|
|
48
|
+
return "https://api.iterable.com/api/"
|
|
49
|
+
|
|
44
50
|
@property
|
|
45
51
|
def retry_factor(self) -> int:
|
|
46
52
|
return 20
|
|
@@ -407,7 +413,7 @@ class CampaignsMetrics(IterableStream):
|
|
|
407
413
|
return params
|
|
408
414
|
|
|
409
415
|
def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]:
|
|
410
|
-
lists = Campaigns(authenticator=self._cred)
|
|
416
|
+
lists = Campaigns(authenticator=self._cred, region=self._region)
|
|
411
417
|
campaign_ids = []
|
|
412
418
|
for list_record in lists.read_records(sync_mode=kwargs.get("sync_mode", SyncMode.full_refresh)):
|
|
413
419
|
campaign_ids.append(list_record["id"])
|