omnata-plugin-runtime 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnata_plugin_runtime/__init__.py +39 -0
- omnata_plugin_runtime/api.py +73 -0
- omnata_plugin_runtime/configuration.py +593 -0
- omnata_plugin_runtime/forms.py +306 -0
- omnata_plugin_runtime/logging.py +91 -0
- omnata_plugin_runtime/omnata_plugin.py +1154 -0
- omnata_plugin_runtime/plugin_entrypoints.py +286 -0
- omnata_plugin_runtime/rate_limiting.py +232 -0
- omnata_plugin_runtime/record_transformer.py +50 -0
- omnata_plugin_runtime-0.1.0.dist-info/LICENSE +504 -0
- omnata_plugin_runtime-0.1.0.dist-info/METADATA +28 -0
- omnata_plugin_runtime-0.1.0.dist-info/RECORD +13 -0
- omnata_plugin_runtime-0.1.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,593 @@
|
|
1
|
+
# it's not the 1980s anymore
|
2
|
+
# pylint: disable=line-too-long,multiple-imports,logging-fstring-interpolation
|
3
|
+
"""
|
4
|
+
Omnata Plugin Runtime configuration objects.
|
5
|
+
Includes data container classes related to plugin configuration.
|
6
|
+
"""
|
7
|
+
from __future__ import annotations
|
8
|
+
import sys
|
9
|
+
from typing import Any, List,Dict,Literal, Union, Optional
|
10
|
+
if tuple(sys.version_info[:2]) >= (3, 9):
|
11
|
+
# Python 3.9 and above
|
12
|
+
from typing import Annotated
|
13
|
+
else:
|
14
|
+
# Python 3.8 and below
|
15
|
+
from typing_extensions import Annotated
|
16
|
+
from abc import ABC
|
17
|
+
from pydantic import BaseModel,Field # pylint: disable=no-name-in-module
|
18
|
+
from enum import Enum
|
19
|
+
|
20
|
+
class MapperType(str, Enum):
|
21
|
+
FIELD_MAPPING_SELECTOR = 'field_mapping_selector'
|
22
|
+
JINJA_TEMPLATE = 'jinja_template'
|
23
|
+
|
24
|
+
class SyncDirection(str, Enum):
|
25
|
+
INBOUND = 'inbound'
|
26
|
+
OUTBOUND = 'outbound'
|
27
|
+
|
28
|
+
class InboundStorageBehaviour(str, Enum):
|
29
|
+
APPEND = 'append'
|
30
|
+
MERGE = 'merge'
|
31
|
+
REPLACE = 'replace'
|
32
|
+
|
33
|
+
class InboundSyncStrategy(str, Enum):
|
34
|
+
FULL_REFRESH = 'Full Refresh'
|
35
|
+
INCREMENTAL = 'Incremental'
|
36
|
+
|
37
|
+
ICON_URL_CODE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="m8 18l-6-6l6-6l1.425 1.425l-4.6 4.6L9.4 16.6Zm8 0l-1.425-1.425l4.6-4.6L14.6 7.4L16 6l6 6Z"/></svg>'
|
38
|
+
ICON_URL_ADD='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="M11 19v-6H5v-2h6V5h2v6h6v2h-6v6Z"/></svg>'
|
39
|
+
ICON_URL_MERGE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><g transform="rotate(90 12 12)"><path fill="currentColor" d="M7.4 20L6 18.6l5-5V6.875L8.425 9.45L7 8.025l5-5l5.025 5.025L15.6 9.475l-2.6-2.6V14.4Zm9.2.025l-3.2-3.175l1.425-1.425l3.175 3.2Z"/></g></svg>'
|
40
|
+
ICON_URL_CALL_MADE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="M5.4 20L4 18.6L15.6 7H9V5h10v10h-2V8.4Z"/></svg>'
|
41
|
+
ICON_URL_DELETE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="m9.4 16.5l2.6-2.6l2.6 2.6l1.4-1.4l-2.6-2.6L16 9.9l-1.4-1.4l-2.6 2.6l-2.6-2.6L8 9.9l2.6 2.6L8 15.1ZM7 21q-.825 0-1.412-.587Q5 19.825 5 19V6H4V4h5V3h6v1h5v2h-1v13q0 .825-.587 1.413Q17.825 21 17 21ZM17 6H7v13h10ZM7 6v13Z"/></svg>'
|
42
|
+
ICON_URL_BASELINE_MERGE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="M6.41 21L5 19.59l4.83-4.83c.75-.75 1.17-1.77 1.17-2.83v-5.1L9.41 8.41L8 7l4-4l4 4l-1.41 1.41L13 6.83v5.1c0 1.06.42 2.08 1.17 2.83L19 19.59L17.59 21L12 15.41L6.41 21z"/></svg>'
|
43
|
+
ICON_URL_REPLACE='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><g fill="none" stroke="currentColor" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"><rect width="6" height="6" x="3" y="3" rx="1"/><rect width="6" height="6" x="15" y="15" rx="1"/><path d="M21 11V8a2 2 0 0 0-2-2h-6l3 3m0-6l-3 3M3 13v3a2 2 0 0 0 2 2h6l-3-3m0 6l3-3"/></g></svg>'
|
44
|
+
ICON_URL_REFRESH='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="M12 20q-3.35 0-5.675-2.325Q4 15.35 4 12q0-3.35 2.325-5.675Q8.65 4 12 4q1.725 0 3.3.713q1.575.712 2.7 2.037V4h2v7h-7V9h4.2q-.8-1.4-2.187-2.2Q13.625 6 12 6Q9.5 6 7.75 7.75T6 12q0 2.5 1.75 4.25T12 18q1.925 0 3.475-1.1T17.65 14h2.1q-.7 2.65-2.85 4.325Q14.75 20 12 20Z"/></svg>'
|
45
|
+
ICON_URL_ADD_ROW='<svg xmlns="http://www.w3.org/2000/svg" width="1em" height="1em" preserveAspectRatio="xMidYMid meet" viewBox="0 0 24 24"><path fill="currentColor" d="M22 10a2 2 0 0 1-2 2H4a2 2 0 0 1-2-2V3h2v2h4V3h2v2h4V3h2v2h4V3h2v7M4 10h4V7H4v3m6 0h4V7h-4v3m10 0V7h-4v3h4m-9 4h2v3h3v2h-3v3h-2v-3H8v-2h3v-3Z"/></svg>'
|
46
|
+
|
47
|
+
class SubscriptableBaseModel(BaseModel):
|
48
|
+
"""
|
49
|
+
Extends the Pydantic BaseModel to make it subscriptable
|
50
|
+
"""
|
51
|
+
def __getitem__(self, item):
|
52
|
+
"""Gets the attribute"""
|
53
|
+
return getattr(self, item)
|
54
|
+
|
55
|
+
class OutboundSyncAction(SubscriptableBaseModel,ABC):
|
56
|
+
"""
|
57
|
+
Base class for Outbound Sync Actions.
|
58
|
+
Describes what action will be taken by the plugin with a particular record.
|
59
|
+
These actions are linked to via sync strategies in response to a record create/update/delete.
|
60
|
+
"""
|
61
|
+
action_name:str
|
62
|
+
description:str
|
63
|
+
custom_action:bool=True
|
64
|
+
|
65
|
+
def __eq__(self, other):
|
66
|
+
if isinstance(other, self.__class__):
|
67
|
+
return self.custom_action == other.custom_action and self.action_name == other.action_name
|
68
|
+
else:
|
69
|
+
return False
|
70
|
+
|
71
|
+
def __init__(self, **data):
|
72
|
+
"""
|
73
|
+
Initialises an OutboundSyncStrategy
|
74
|
+
"""
|
75
|
+
if 'action_name' not in data:
|
76
|
+
raise ValueError("'action_name' not set on OutboundSyncStrategy object")
|
77
|
+
if 'custom_action' in data and data['custom_action'] is False:
|
78
|
+
# The built-in outbound sync strategies do not need to have their entire contents stored, just the name
|
79
|
+
if self.__class__ != STANDARD_OUTBOUND_SYNC_ACTIONS[data['action_name']]:
|
80
|
+
data = {**data,**STANDARD_OUTBOUND_SYNC_ACTIONS[data['action_name']]().__dict__}
|
81
|
+
super().__init__(**data)
|
82
|
+
|
83
|
+
def dict(self, *args,trim:bool=True,**kwargs) -> Dict[str,any]:
|
84
|
+
"""
|
85
|
+
OutboundSyncStrategy objects can be serialized much smaller if the instance is of a Standard strategy that resides in the
|
86
|
+
Omnata plugin runtime. This method trims out the unneeded attributes in these scenarios.
|
87
|
+
This method also trims the actions for this strategy if it's a custom one.
|
88
|
+
|
89
|
+
"""
|
90
|
+
excluded_fields = {} if not trim or self.custom_action else {'description'}
|
91
|
+
return super().dict(exclude=excluded_fields)
|
92
|
+
|
93
|
+
class CreateSyncAction(OutboundSyncAction):
|
94
|
+
"""
|
95
|
+
The standard Create sync action.
|
96
|
+
Indicates that a record/object will be created in the target app.
|
97
|
+
"""
|
98
|
+
def __init__(self):
|
99
|
+
OutboundSyncAction.__init__(self,action_name='Create',
|
100
|
+
description='Object will be created in the app',
|
101
|
+
custom_action=False)
|
102
|
+
|
103
|
+
class UpdateSyncAction(OutboundSyncAction):
|
104
|
+
"""
|
105
|
+
The standard Update sync action.
|
106
|
+
Indicates that a record/object will be updated in the target app.
|
107
|
+
"""
|
108
|
+
def __init__(self):
|
109
|
+
OutboundSyncAction.__init__(self,action_name='Update',
|
110
|
+
description='Object will be updated in the app',
|
111
|
+
custom_action=False)
|
112
|
+
|
113
|
+
class DeleteSyncAction(OutboundSyncAction):
|
114
|
+
"""
|
115
|
+
The standard Delete sync action.
|
116
|
+
Indicates that a record/object will be deleted in the target app.
|
117
|
+
"""
|
118
|
+
def __init__(self):
|
119
|
+
OutboundSyncAction.__init__(self,action_name='Delete',
|
120
|
+
description='Object will be deleted in the app',
|
121
|
+
custom_action=False)
|
122
|
+
|
123
|
+
class SendSyncAction(OutboundSyncAction):
|
124
|
+
"""
|
125
|
+
The standard Send sync action.
|
126
|
+
Indicates that a record/object will be sent to the target app. This action is typically
|
127
|
+
used in event-style applications, to indicate that the action is more of a message than a record operation.
|
128
|
+
"""
|
129
|
+
def __init__(self):
|
130
|
+
OutboundSyncAction.__init__(self,action_name='Send',
|
131
|
+
description='Record will be sent to the app',
|
132
|
+
custom_action=False)
|
133
|
+
|
134
|
+
class RecreateSyncAction(OutboundSyncAction):
|
135
|
+
"""
|
136
|
+
The standard Recreate sync action.
|
137
|
+
Similar to the Create action, but indicates that all existing records in the app will be replaced by these.
|
138
|
+
If a sync strategy uses this sync action, it doesn't make sense to include any other actions.
|
139
|
+
"""
|
140
|
+
def __init__(self):
|
141
|
+
OutboundSyncAction.__init__(self,action_name='Recreate',
|
142
|
+
description='Record(s) will replace any currently in the app',
|
143
|
+
custom_action=False)
|
144
|
+
|
145
|
+
STANDARD_OUTBOUND_SYNC_ACTIONS:Dict[str,OutboundSyncAction] = {
|
146
|
+
'Create': CreateSyncAction,
|
147
|
+
'Update': UpdateSyncAction,
|
148
|
+
'Delete': DeleteSyncAction,
|
149
|
+
'Send': SendSyncAction,
|
150
|
+
'Recreate': RecreateSyncAction
|
151
|
+
}
|
152
|
+
|
153
|
+
class OutboundSyncStrategy(SubscriptableBaseModel,ABC):
|
154
|
+
"""OutboundSyncStrategy is a base class for all outbound sync strategies.
|
155
|
+
Each implementation decides on what pattern of record changes it needs to observe.
|
156
|
+
For each type of record change, an OutboundSyncAction describes what it will do in the target app.
|
157
|
+
|
158
|
+
Custom OutboundSyncStrategies can be devised, which provide for use cases beyond applying records
|
159
|
+
and publishing events.
|
160
|
+
|
161
|
+
"""
|
162
|
+
name:str
|
163
|
+
description:str
|
164
|
+
icon_source:str=ICON_URL_CODE
|
165
|
+
action_on_record_create:OutboundSyncAction=None
|
166
|
+
action_on_record_update:OutboundSyncAction=None
|
167
|
+
action_on_record_delete:OutboundSyncAction=None
|
168
|
+
action_on_record_unchanged:OutboundSyncAction=None
|
169
|
+
custom_strategy:bool=True
|
170
|
+
|
171
|
+
def __eq__(self, other):
|
172
|
+
if isinstance(other, self.__class__):
|
173
|
+
return self.custom_strategy == other.custom_strategy and self.name == other.name
|
174
|
+
else:
|
175
|
+
return False
|
176
|
+
|
177
|
+
def __init__(self, **data):
|
178
|
+
"""
|
179
|
+
Initialises an OutboundSyncStrategy
|
180
|
+
"""
|
181
|
+
if 'name' not in data:
|
182
|
+
raise ValueError("'name' not set on OutboundSyncStrategy object")
|
183
|
+
if 'custom_strategy' in data and data['custom_strategy'] is False:
|
184
|
+
# The built-in outbound sync strategies do not need to have their entire contents stored, just the name
|
185
|
+
if self.__class__ != STANDARD_OUTBOUND_SYNC_STRATEGIES[data['name']]:
|
186
|
+
data = {**data,**STANDARD_OUTBOUND_SYNC_STRATEGIES[data['name']]().__dict__}
|
187
|
+
super().__init__(**data)
|
188
|
+
|
189
|
+
def dict(self, *args,trim:bool=True,**kwargs) -> Dict[str,any]:
|
190
|
+
"""
|
191
|
+
OutboundSyncStrategy objects can be serialized much smaller if the instance is of a Standard strategy that resides in the
|
192
|
+
Omnata plugin runtime. This method trims out the unneeded attributes in these scenarios.
|
193
|
+
This method also trims the actions for this strategy if it's a custom one.
|
194
|
+
|
195
|
+
"""
|
196
|
+
excluded_fields = {} if not trim or self.custom_strategy else {'description','icon_source','action_on_record_create','action_on_record_update','action_on_record_delete','action_on_record_unchanged'}
|
197
|
+
return super().dict(exclude=excluded_fields)
|
198
|
+
|
199
|
+
|
200
|
+
class CreateSyncStrategy(OutboundSyncStrategy):
|
201
|
+
"""
|
202
|
+
The standard Create sync strategy.
|
203
|
+
Record creation -> CreateSyncAction
|
204
|
+
"""
|
205
|
+
def __init__(self):
|
206
|
+
OutboundSyncStrategy.__init__(self, name='Create', \
|
207
|
+
description='Creates new objects only, does not update or delete', \
|
208
|
+
action_on_record_create=CreateSyncAction(), \
|
209
|
+
icon_source=ICON_URL_ADD,
|
210
|
+
custom_strategy=False)
|
211
|
+
|
212
|
+
class UpsertSyncStrategy(OutboundSyncStrategy):
|
213
|
+
"""
|
214
|
+
The standard Upsert sync strategy.
|
215
|
+
Record creation -> CreateSyncAction
|
216
|
+
Record update -> UpdateSyncAction
|
217
|
+
"""
|
218
|
+
def __init__(self):
|
219
|
+
OutboundSyncStrategy.__init__(self, name='Upsert', \
|
220
|
+
description='Creates new objects, updates existing objects, does not delete', \
|
221
|
+
action_on_record_create=CreateSyncAction(), \
|
222
|
+
action_on_record_update=UpdateSyncAction(), \
|
223
|
+
icon_source=ICON_URL_MERGE,
|
224
|
+
custom_strategy=False)
|
225
|
+
|
226
|
+
class UpdateSyncStrategy(OutboundSyncStrategy):
|
227
|
+
"""
|
228
|
+
The standard Update sync strategy.
|
229
|
+
Record update -> UpdateSyncAction
|
230
|
+
"""
|
231
|
+
def __init__(self):
|
232
|
+
OutboundSyncStrategy.__init__(self, name='Update', \
|
233
|
+
description='Updates existing objects only', \
|
234
|
+
action_on_record_update=UpdateSyncAction(), \
|
235
|
+
icon_source=ICON_URL_CALL_MADE,
|
236
|
+
custom_strategy=False)
|
237
|
+
|
238
|
+
class DeleteSyncStrategy(OutboundSyncStrategy):
|
239
|
+
"""
|
240
|
+
The standard Delete sync strategy.
|
241
|
+
Record deletion -> DeleteSyncAction
|
242
|
+
"""
|
243
|
+
def __init__(self):
|
244
|
+
OutboundSyncStrategy.__init__(self, name='Delete', \
|
245
|
+
description='Deletes objects as they appear in the source', \
|
246
|
+
action_on_record_create=DeleteSyncAction(), \
|
247
|
+
icon_source=ICON_URL_DELETE,
|
248
|
+
custom_strategy=False)
|
249
|
+
|
250
|
+
class MirrorSyncStrategy(OutboundSyncStrategy):
|
251
|
+
"""
|
252
|
+
The standard Mirror sync strategy.
|
253
|
+
Record creation -> CreateSyncAction
|
254
|
+
Record update -> UpdateSyncAction
|
255
|
+
Record delete -> DeleteSyncAction
|
256
|
+
"""
|
257
|
+
def __init__(self):
|
258
|
+
OutboundSyncStrategy.__init__(self, name='Mirror', \
|
259
|
+
description='Creates new objects, updates existing objects, deletes when removed', \
|
260
|
+
action_on_record_create=CreateSyncAction(), \
|
261
|
+
action_on_record_update=UpdateSyncAction(), \
|
262
|
+
action_on_record_delete=DeleteSyncAction(), \
|
263
|
+
icon_source=ICON_URL_BASELINE_MERGE,
|
264
|
+
custom_strategy=False)
|
265
|
+
|
266
|
+
class SendSyncStrategy(OutboundSyncStrategy):
|
267
|
+
"""
|
268
|
+
The standard Send sync strategy.
|
269
|
+
Record creation -> SendSyncAction
|
270
|
+
"""
|
271
|
+
def __init__(self):
|
272
|
+
OutboundSyncStrategy.__init__(self, name='Send', \
|
273
|
+
description='Sends new objects. Similar to create, but intended for event-style rather than record-style syncs', \
|
274
|
+
action_on_record_create=SendSyncAction(), \
|
275
|
+
icon_source=ICON_URL_ADD,
|
276
|
+
custom_strategy=False)
|
277
|
+
|
278
|
+
class ReplaceSyncStrategy(OutboundSyncStrategy):
|
279
|
+
"""
|
280
|
+
The standard Replace sync strategy.
|
281
|
+
This is a special strategy that means all records that currently exist will be recreated each sync.
|
282
|
+
Record creation -> RecreateSyncAction
|
283
|
+
Record update -> RecreateSyncAction
|
284
|
+
Record unchanged -> RecreateSyncAction
|
285
|
+
"""
|
286
|
+
def __init__(self):
|
287
|
+
OutboundSyncStrategy.__init__(self, name='Replace', \
|
288
|
+
description='Applies all current records, regardless of history', \
|
289
|
+
action_on_record_create=RecreateSyncAction(),
|
290
|
+
action_on_record_update=RecreateSyncAction(),
|
291
|
+
action_on_record_unchanged=RecreateSyncAction(),
|
292
|
+
icon_source=ICON_URL_REPLACE,
|
293
|
+
custom_strategy=False)
|
294
|
+
|
295
|
+
STANDARD_OUTBOUND_SYNC_STRATEGIES: Dict[str,OutboundSyncStrategy] = {
|
296
|
+
'Create': CreateSyncStrategy,
|
297
|
+
'Upsert': UpsertSyncStrategy,
|
298
|
+
'Update': UpdateSyncStrategy,
|
299
|
+
'Delete': DeleteSyncStrategy,
|
300
|
+
'Mirror': MirrorSyncStrategy,
|
301
|
+
'Replace': ReplaceSyncStrategy,
|
302
|
+
'Send': SendSyncStrategy
|
303
|
+
}
|
304
|
+
|
305
|
+
class InboundSyncStreamsConfiguration(SubscriptableBaseModel):
|
306
|
+
"""
|
307
|
+
Encapsulates the whole value stored under STREAMS_CONFIGURATION. Includes configuration of streams,
|
308
|
+
as well as which ones were excluded and how to treat newly discovered objects
|
309
|
+
"""
|
310
|
+
include_new_streams:bool
|
311
|
+
new_stream_sync_strategy: Optional[InboundSyncStrategy]
|
312
|
+
new_stream_storage_behaviour:Optional[InboundStorageBehaviour]
|
313
|
+
included_streams: Dict[str, StoredStreamConfiguration]
|
314
|
+
excluded_streams: List[str]
|
315
|
+
|
316
|
+
class StoredStreamConfiguration(SubscriptableBaseModel):
|
317
|
+
"""
|
318
|
+
Encapsulates all of the configuration necessary to sync an inbound stream.
|
319
|
+
This information is parsed from the metadata of the streams Sync config object, for convenience.
|
320
|
+
"""
|
321
|
+
stream_name:str
|
322
|
+
sync_strategy:InboundSyncStrategy
|
323
|
+
cursor_field: Optional[str] = Field(
|
324
|
+
None,
|
325
|
+
description="The field to use as a cursor",
|
326
|
+
)
|
327
|
+
# Composite primary keys is not really a thing in SaaS applications
|
328
|
+
primary_key_field: Optional[str] = Field(
|
329
|
+
None,
|
330
|
+
description="The field that will be used as primary key.",
|
331
|
+
)
|
332
|
+
storage_behaviour:InboundStorageBehaviour
|
333
|
+
stream: StreamConfiguration
|
334
|
+
latest_state:dict = Field(default_factory=dict)
|
335
|
+
|
336
|
+
class StreamConfiguration(SubscriptableBaseModel):
|
337
|
+
"""
|
338
|
+
Encapsulates all of the configuration necessary to sync an inbound stream.
|
339
|
+
Derived from the Airbyte protocol, with minor tweaks to suit our differences.
|
340
|
+
"""
|
341
|
+
stream_name:str
|
342
|
+
supported_sync_strategies:List[InboundSyncStrategy]
|
343
|
+
source_defined_cursor:Optional[bool] = Field(
|
344
|
+
None,
|
345
|
+
description="If true, the plugin controls the cursor field",
|
346
|
+
)
|
347
|
+
source_defined_primary_key:Optional[str] = Field(
|
348
|
+
None,
|
349
|
+
description="If true, the plugin controls the primary key",
|
350
|
+
)
|
351
|
+
default_cursor_field:Optional[str] = Field(
|
352
|
+
None,
|
353
|
+
description="The default field to use as a cursor",
|
354
|
+
)
|
355
|
+
json_schema:Optional[Dict[str, Any]] = Field(None, description="JSON Schema for objects provided by stream")
|
356
|
+
|
357
|
+
class StoredConfigurationValue(SubscriptableBaseModel):
|
358
|
+
"""
|
359
|
+
A configuration value that was provided by a user (to configure a sync or connection).
|
360
|
+
It contains only a string value and optionally some metadata, all of the display-related properties are discarded
|
361
|
+
"""
|
362
|
+
value:str = Field(
|
363
|
+
description="The stored value",
|
364
|
+
)
|
365
|
+
metadata:dict = Field(
|
366
|
+
default_factory=dict,
|
367
|
+
description="Metadata associated with the value")
|
368
|
+
|
369
|
+
class ConnectionConfigurationParameters(SubscriptableBaseModel):
|
370
|
+
"""
|
371
|
+
Contains user-provided information completed during connection configuration.
|
372
|
+
This acts as a base class since connection parameters are the first things collected.
|
373
|
+
"""
|
374
|
+
connection_method:str
|
375
|
+
connection_parameters:Dict[str, StoredConfigurationValue] = None
|
376
|
+
connection_secrets:Dict[str, StoredConfigurationValue] = None
|
377
|
+
|
378
|
+
def get_connection_parameter(self,parameter_name:str) -> StoredConfigurationValue:
|
379
|
+
"""
|
380
|
+
Retrieves a connection parameter, which was collected during connection configuration.
|
381
|
+
What you can expect to retrieve is based on the form definition returned by your connection_form function (form fields returned with secret=False).
|
382
|
+
|
383
|
+
:param str parameter_name: The name of the parameter
|
384
|
+
:return: the configuration value, which contains a string property named "value", and a metadata dict
|
385
|
+
:rtype: StoredConfigurationValue
|
386
|
+
:raises ValueError: if a connection parameter by that name does not exist
|
387
|
+
"""
|
388
|
+
if self.connection_parameters is None:
|
389
|
+
raise ValueError("Connection parameters were not provided")
|
390
|
+
if parameter_name not in self.connection_parameters.keys():
|
391
|
+
raise ValueError(f"Connection parameter '{parameter_name}' not available")
|
392
|
+
return self.connection_parameters[parameter_name] #pylint: disable=unsubscriptable-object
|
393
|
+
|
394
|
+
def get_connection_secret(self,parameter_name:str) -> StoredConfigurationValue:
|
395
|
+
"""
|
396
|
+
Retrieves a connection secret, which was collected during connection configuration.
|
397
|
+
What you can expect to retrieve is based on the form definition returned by your connection_form function (form fields returned with secret=False).
|
398
|
+
|
399
|
+
:param str parameter_name: The name of the parameter
|
400
|
+
:return: the configuration value, which contains a string property named "value", and a metadata dict
|
401
|
+
:rtype: StoredConfigurationValue
|
402
|
+
:raises ValueError: if a connection parameter by that name does not exist
|
403
|
+
"""
|
404
|
+
if self.connection_secrets is None:
|
405
|
+
raise ValueError("Connection secrets were not provided")
|
406
|
+
if parameter_name not in self.connection_secrets.keys():
|
407
|
+
raise ValueError(f"Connection secret '{parameter_name}' not available")
|
408
|
+
return self.connection_secrets[parameter_name] #pylint: disable=unsubscriptable-object
|
409
|
+
|
410
|
+
class SyncConfigurationParameters(ConnectionConfigurationParameters):
|
411
|
+
"""
|
412
|
+
A base class for Sync configuration parameters.
|
413
|
+
"""
|
414
|
+
connection_method:str
|
415
|
+
connection_parameters:Dict[str, StoredConfigurationValue] = None
|
416
|
+
connection_secrets:Dict[str, StoredConfigurationValue] = None
|
417
|
+
sync_parameters:Dict[str, StoredConfigurationValue] = None
|
418
|
+
current_form_parameters:Dict[str, StoredConfigurationValue] = None
|
419
|
+
|
420
|
+
def sync_parameter_exists(self,parameter_name:str) -> bool:
|
421
|
+
"""
|
422
|
+
Advises whether or not a sync parameter exists.
|
423
|
+
|
424
|
+
:param str parameter_name: The name of the parameter
|
425
|
+
:return: True if the given parameter exists, otherwise False
|
426
|
+
:rtype: bool
|
427
|
+
"""
|
428
|
+
return parameter_name in self.sync_parameters.keys()
|
429
|
+
|
430
|
+
def get_sync_parameter(self,parameter_name:str,default_value:str=None) -> StoredConfigurationValue:
|
431
|
+
"""
|
432
|
+
Retrieves a sync parameter, which was collected during sync configuration.
|
433
|
+
What you can expect to retrieve is based on the form definition returned by your configuration function.
|
434
|
+
|
435
|
+
:param str parameter_name: The name of the parameter
|
436
|
+
:param str default_value: A default value to return (under the "value" property) if the parameter does not exist
|
437
|
+
:return: the configuration value, which contains a string property named "value", and a metadata dict
|
438
|
+
:rtype: StoredConfigurationValue
|
439
|
+
:raises ValueError: if a sync parameter by that name does not exist, and no default was provided
|
440
|
+
"""
|
441
|
+
if parameter_name not in self.sync_parameters.keys():
|
442
|
+
if default_value is not None:
|
443
|
+
return {"value":default_value,"metadata":{}}
|
444
|
+
raise ValueError(f"Sync parameter '{parameter_name}' not available")
|
445
|
+
return self.sync_parameters[parameter_name] #pylint: disable=unsubscriptable-object
|
446
|
+
|
447
|
+
def get_current_form_parameter(self,parameter_name:str,default_value:str=None) -> StoredConfigurationValue:
|
448
|
+
"""
|
449
|
+
Retrieves a connection parameter, which was collected during connection configuration.
|
450
|
+
What you can expect to retrieve is based on the form definition returned by your connection_form function (form fields returned with secret=False).
|
451
|
+
|
452
|
+
:param str parameter_name: The name of the parameter
|
453
|
+
:param str default_value: A default value to return if the parameter does not exist
|
454
|
+
:return: the configuration value, which contains a string property named "value", and a metadata dict
|
455
|
+
:rtype: StoredConfigurationValue
|
456
|
+
:raises ValueError: if a connection parameter by that name does not exist
|
457
|
+
"""
|
458
|
+
if parameter_name not in self.current_form_parameters.keys():
|
459
|
+
if default_value is not None:
|
460
|
+
return StoredConfigurationValue(value=default_value)
|
461
|
+
raise ValueError(f"Form parameter '{parameter_name}' not available")
|
462
|
+
return self.current_form_parameters[parameter_name] #pylint: disable=unsubscriptable-object
|
463
|
+
|
464
|
+
def condense_current_form_parameters(self,exclude_fields:List[str]) -> dict:
|
465
|
+
"""
|
466
|
+
Takes a dictionary representing a completed form, and condenses it into a simple dictionary containing just the values of each field
|
467
|
+
"""
|
468
|
+
return_dict={}
|
469
|
+
for dict_key in self.current_form_parameters.keys():
|
470
|
+
if dict_key not in exclude_fields:
|
471
|
+
return_dict[dict_key] = self.current_form_parameters[dict_key].value #pylint: disable=unsubscriptable-object
|
472
|
+
return return_dict
|
473
|
+
|
474
|
+
class OutboundSyncConfigurationParameters(SyncConfigurationParameters):
|
475
|
+
"""
|
476
|
+
Contains user-provided information completed during outbound connection/sync configuration.
|
477
|
+
"""
|
478
|
+
connection_method:str
|
479
|
+
connection_parameters:Dict[str, StoredConfigurationValue] = None
|
480
|
+
connection_secrets:Dict[str, StoredConfigurationValue] = None
|
481
|
+
sync_parameters:Dict[str, StoredConfigurationValue] = None
|
482
|
+
current_form_parameters:Dict[str, StoredConfigurationValue] = None
|
483
|
+
sync_strategy:OutboundSyncStrategy = None
|
484
|
+
field_mappings:StoredMappingValue = None
|
485
|
+
|
486
|
+
class InboundSyncConfigurationParameters(SyncConfigurationParameters):
|
487
|
+
"""
|
488
|
+
Contains user-provided information completed during inbound connection/sync configuration.
|
489
|
+
"""
|
490
|
+
connection_method:str
|
491
|
+
connection_parameters:Dict[str, StoredConfigurationValue] = None
|
492
|
+
connection_secrets:Dict[str, StoredConfigurationValue] = None
|
493
|
+
sync_parameters:Dict[str, StoredConfigurationValue] = None
|
494
|
+
current_form_parameters:Dict[str, StoredConfigurationValue] = None
|
495
|
+
#sync_strategy:InboundSyncStrategy = None
|
496
|
+
|
497
|
+
class StoredJinjaTemplate(SubscriptableBaseModel):
|
498
|
+
"""
|
499
|
+
Mapping information that was provided by a user (to configure a sync or connection).
|
500
|
+
It contains either a list of mappings or a jinja template
|
501
|
+
"""
|
502
|
+
mapper_type:Literal["jinja_template"] = 'jinja_template'
|
503
|
+
jinja_template:str
|
504
|
+
|
505
|
+
class StoredFieldMappings(SubscriptableBaseModel):
|
506
|
+
"""
|
507
|
+
Mapping information that was provided by a user (to configure a sync or connection).
|
508
|
+
It contains either a list of mappings or a jinja template
|
509
|
+
"""
|
510
|
+
mapper_type:Literal["field_mapping_selector"] = 'field_mapping_selector'
|
511
|
+
field_mappings:List[StoredFieldMapping]
|
512
|
+
|
513
|
+
StoredMappingValue = Annotated[Union[StoredJinjaTemplate,StoredFieldMappings],Field(discriminator='mapper_type')]
|
514
|
+
|
515
|
+
class SyncScheduleSnowflakeTask(SubscriptableBaseModel):
|
516
|
+
"""
|
517
|
+
A sync schedule which uses Snowflake tasks to initiate runs.
|
518
|
+
|
519
|
+
Args:
|
520
|
+
sync_frequency (str): The cron expression for this schedule
|
521
|
+
sync_frequency_name (str): The plain english name for this cron schedule
|
522
|
+
warehouse (str): The Snowflake warehouse which this task uses
|
523
|
+
daily_hour (Optional[int]): If the sync frequency is Daily, the hour it runs
|
524
|
+
daily_minute (Optional[int]): If the sync frequency is Daily, the minute it runs
|
525
|
+
minute_of_hour (Optional[int]): If the sync frequency is Hourly, the minute of the hour it runs
|
526
|
+
|
527
|
+
"""
|
528
|
+
mode:Literal['snowflake_task']
|
529
|
+
sync_frequency:str
|
530
|
+
sync_frequency_name:Literal['1 min', '5 mins', '15 mins', 'Hourly', 'Daily']
|
531
|
+
warehouse:str
|
532
|
+
daily_hour:Optional[int] = None
|
533
|
+
daily_minute:Optional[int] = None
|
534
|
+
minute_of_hour:Optional[int] = None
|
535
|
+
|
536
|
+
class SyncScheduleDbt(SubscriptableBaseModel):
|
537
|
+
"""
|
538
|
+
A sync schedule which runs when initiated during a dbt run, using the Omnata dbt package.
|
539
|
+
|
540
|
+
Args:
|
541
|
+
dbt_prod_target_name (str): The name of the dbt target used for production runs
|
542
|
+
warehouse (str): The Snowflake warehouse which this task uses
|
543
|
+
dbt_dev_target_name (str): The name of the dbt target used by developers outside of CI
|
544
|
+
dbt_sync_model_name (str): The name of the dbt model used to run the sync
|
545
|
+
dbt_source_model_name (str): The name of the dbt model used as a source for the run. This will be resolved at runtime and replaces the configured source table
|
546
|
+
is_dbt_cloud (bool): If true, dbt cloud is in use
|
547
|
+
"""
|
548
|
+
mode:Literal['dbt']
|
549
|
+
dbt_prod_target_name:str = 'prod'
|
550
|
+
warehouse: str
|
551
|
+
dbt_dev_target_name:str = 'dev'
|
552
|
+
dbt_sync_model_name:str
|
553
|
+
dbt_source_model_name:str
|
554
|
+
is_dbt_cloud:bool = True
|
555
|
+
|
556
|
+
class SyncScheduleDependant(SubscriptableBaseModel):
|
557
|
+
"""
|
558
|
+
A sync schedule which runs the sync at the same time or after another sync.
|
559
|
+
|
560
|
+
Args:
|
561
|
+
run_when (Literal["after_parent_completes","at_same_time_as"]): Determines if this sync runs at the same time as the parent, or after it completes
|
562
|
+
warehouse (str): The Snowflake warehouse which this task uses
|
563
|
+
selected_sync (int): The sync ID of the sync to depend on
|
564
|
+
"""
|
565
|
+
mode:Literal['dependent']
|
566
|
+
run_when:Literal["after_parent_completes","at_same_time_as"] = 'after_parent_completes'
|
567
|
+
warehouse:str
|
568
|
+
selected_sync:int
|
569
|
+
|
570
|
+
class SyncScheduleManual(SubscriptableBaseModel):
|
571
|
+
"""
|
572
|
+
A sync schedule which runs only when manually requested.
|
573
|
+
|
574
|
+
Args:
|
575
|
+
warehouse (str): The Snowflake warehouse which this task uses
|
576
|
+
"""
|
577
|
+
mode:Literal['manual']
|
578
|
+
warehouse:str
|
579
|
+
|
580
|
+
SyncSchedule = Annotated[Union[SyncScheduleSnowflakeTask,SyncScheduleDbt,SyncScheduleDependant,SyncScheduleManual],Field(discriminator='mode')]
|
581
|
+
|
582
|
+
class StoredFieldMapping(SubscriptableBaseModel):
|
583
|
+
"""
|
584
|
+
A column->field mapping value that was provided by a user when configuring a sync.
|
585
|
+
"""
|
586
|
+
source_column:str
|
587
|
+
app_field:str
|
588
|
+
app_metadata:dict = Field(default_factory=dict)
|
589
|
+
|
590
|
+
StoredStreamConfiguration.update_forward_refs()
|
591
|
+
InboundSyncStreamsConfiguration.update_forward_refs()
|
592
|
+
StoredFieldMappings.update_forward_refs()
|
593
|
+
OutboundSyncConfigurationParameters.update_forward_refs()
|