omnata-plugin-runtime 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- omnata_plugin_runtime/__init__.py +39 -0
- omnata_plugin_runtime/api.py +73 -0
- omnata_plugin_runtime/configuration.py +593 -0
- omnata_plugin_runtime/forms.py +306 -0
- omnata_plugin_runtime/logging.py +91 -0
- omnata_plugin_runtime/omnata_plugin.py +1154 -0
- omnata_plugin_runtime/plugin_entrypoints.py +286 -0
- omnata_plugin_runtime/rate_limiting.py +232 -0
- omnata_plugin_runtime/record_transformer.py +50 -0
- omnata_plugin_runtime-0.1.0.dist-info/LICENSE +504 -0
- omnata_plugin_runtime-0.1.0.dist-info/METADATA +28 -0
- omnata_plugin_runtime-0.1.0.dist-info/RECORD +13 -0
- omnata_plugin_runtime-0.1.0.dist-info/WHEEL +4 -0
@@ -0,0 +1,306 @@
|
|
1
|
+
# it's not the 1980s anymore
|
2
|
+
# pylint: disable=line-too-long,multiple-imports,logging-fstring-interpolation,too-many-arguments,no-self-argument
|
3
|
+
"""
|
4
|
+
Contains form elements for Omnata plugin configuration
|
5
|
+
"""
|
6
|
+
from __future__ import annotations
|
7
|
+
import sys
|
8
|
+
from typing import List,Callable, Literal, Union, ForwardRef, Optional
|
9
|
+
if tuple(sys.version_info[:2]) >= (3, 9):
|
10
|
+
# Python 3.9 and above
|
11
|
+
from typing import Annotated
|
12
|
+
else:
|
13
|
+
# Python 3.8 and below
|
14
|
+
from typing_extensions import Annotated
|
15
|
+
from abc import ABC
|
16
|
+
from types import MethodType
|
17
|
+
from pydantic import BaseModel,Field,validator # pylint: disable=no-name-in-module
|
18
|
+
from .configuration import SubscriptableBaseModel, StreamConfiguration, SyncConfigurationParameters, InboundSyncConfigurationParameters
|
19
|
+
|
20
|
+
class FormOption(SubscriptableBaseModel):
|
21
|
+
"""
|
22
|
+
An option used by certain form forms (like Dropdowns).
|
23
|
+
|
24
|
+
:param str value: The value to set in the field if this option is selected
|
25
|
+
:param str label: The label to show in the list. This value is not stored.
|
26
|
+
:param dict metadata: An arbitrary dictionary to store with the value, which can be retrieved by the plugin
|
27
|
+
:param bool required: When populating field mapping options, this flag indicates that this field is mandatory
|
28
|
+
:param bool unique: When populating field mapping options, this flag indicates that this field requires a unique value (i.e. be mapped from a unique column)
|
29
|
+
:param bool default: Indicates that this option should be the default selected
|
30
|
+
:param bool disabled: Indicates that the option should appear in the list, but be unselectable
|
31
|
+
:param str data_type_icon: The data type icon to show next to the option (where applicable)
|
32
|
+
:return: nothing
|
33
|
+
"""
|
34
|
+
value:str
|
35
|
+
label:str
|
36
|
+
metadata:dict = Field(default_factory=dict)
|
37
|
+
required:bool=False
|
38
|
+
unique:bool=False
|
39
|
+
default:bool=False
|
40
|
+
disabled:bool=False
|
41
|
+
data_type_icon:str='unknown'
|
42
|
+
|
43
|
+
class FormInputField(SubscriptableBaseModel):
|
44
|
+
"""
|
45
|
+
An input field, which collects a single line of free-form text from the user and no metadata.
|
46
|
+
"""
|
47
|
+
name:str
|
48
|
+
label:str
|
49
|
+
default_value:Union[str,bool]=''
|
50
|
+
required:bool=False
|
51
|
+
depends_on:str=None
|
52
|
+
help_text:str=""
|
53
|
+
reload_on_change:bool=False
|
54
|
+
secret:bool=False
|
55
|
+
type:Literal['input'] = 'input'
|
56
|
+
|
57
|
+
class FormTextAreaField(SubscriptableBaseModel):
|
58
|
+
"""
|
59
|
+
A text area field, which collects multi-line free-form text from the user and no metadata.
|
60
|
+
|
61
|
+
:param str name: The name of the form field. This value must be unique, and is used to retrieve the value from within the plugin code.
|
62
|
+
:param str label: The label for the form field
|
63
|
+
:param str default_value: The default value presented initially in the field
|
64
|
+
:param bool required: If True, means that the form cannot be submitted without a value present
|
65
|
+
:param str depends_on: The name of another form field. If provided, this form field will not be rendered until there is a value in the field it depends on.
|
66
|
+
:param str help_text: A longer description of what the field is used for. If provided, a help icon is shown and must be hovered over to display this text.
|
67
|
+
:param bool secret: Indicates that the text entered must be masked in the browser, and stored/access securely
|
68
|
+
:param bool reload_on_change: If True, the entire form is reloaded after the value is changed. This is used to conditionally render fields based on values provided in others, but should be used only when strictly necessary.
|
69
|
+
:return: nothing
|
70
|
+
"""
|
71
|
+
name:str
|
72
|
+
label:str
|
73
|
+
default_value:str=''
|
74
|
+
secret:bool=False
|
75
|
+
required:bool=False
|
76
|
+
depends_on:str=None
|
77
|
+
help_text:str=""
|
78
|
+
reload_on_change:bool=False
|
79
|
+
type:Literal['textarea'] = 'textarea'
|
80
|
+
|
81
|
+
variables:bool=False
|
82
|
+
|
83
|
+
class FormSshKeypair(SubscriptableBaseModel):
|
84
|
+
"""
|
85
|
+
An SSH Keypair field, which generates public and private keys for asymmetric cryptography.
|
86
|
+
"""
|
87
|
+
name:str
|
88
|
+
label:str
|
89
|
+
default_value:str=None
|
90
|
+
required:bool=False
|
91
|
+
depends_on:str=None
|
92
|
+
help_text:str=""
|
93
|
+
|
94
|
+
type:Literal['ssh_keypair'] = 'ssh_keypair'
|
95
|
+
secret:bool=True
|
96
|
+
|
97
|
+
class FormCheckboxField(SubscriptableBaseModel):
|
98
|
+
"""
|
99
|
+
A field which presents a checkbox
|
100
|
+
"""
|
101
|
+
name:str
|
102
|
+
label:str
|
103
|
+
default_value:bool=False
|
104
|
+
required:bool=False
|
105
|
+
secret:bool=False
|
106
|
+
depends_on:str=None
|
107
|
+
help_text:str=""
|
108
|
+
reload_on_change:bool=False
|
109
|
+
type:Literal['checkbox'] = 'checkbox'
|
110
|
+
|
111
|
+
class FormSliderField(SubscriptableBaseModel):
|
112
|
+
"""
|
113
|
+
A field which presents a slider
|
114
|
+
"""
|
115
|
+
name:str
|
116
|
+
label:str
|
117
|
+
default_value:str=None
|
118
|
+
secret:bool=False
|
119
|
+
required:bool=False
|
120
|
+
depends_on:str=None
|
121
|
+
help_text:str=""
|
122
|
+
reload_on_change:bool=False
|
123
|
+
type:Literal['slider'] = 'slider'
|
124
|
+
|
125
|
+
min_value:int=0
|
126
|
+
max_value:int=100
|
127
|
+
step_size:int=1
|
128
|
+
|
129
|
+
class StreamLister(SubscriptableBaseModel):
|
130
|
+
"""
|
131
|
+
A class for managing the listing of Streams. Can depend on other form fields to delay rendering
|
132
|
+
"""
|
133
|
+
source_function:Union[Callable[[InboundSyncConfigurationParameters], List[StreamConfiguration]],str]
|
134
|
+
label:str="Select Objects"
|
135
|
+
depends_on:str=None
|
136
|
+
|
137
|
+
@validator('source_function', always=True)
|
138
|
+
def function_name_convertor(cls, v, values) -> str:
|
139
|
+
return v.__name__ if isinstance(v,MethodType) else v
|
140
|
+
|
141
|
+
class FormJinjaTemplate(SubscriptableBaseModel):
|
142
|
+
"""
|
143
|
+
Uses text area to allow the user to create a template, which can include column values from the source
|
144
|
+
"""
|
145
|
+
mapper_type:Literal["jinja_template"] = 'jinja_template'
|
146
|
+
label:str="Jinja Template"
|
147
|
+
label:str=None
|
148
|
+
depends_on:str=None
|
149
|
+
|
150
|
+
# ----------------------------------------------------------------------------
|
151
|
+
# Everything above here has no dependencies on other BaseModels in this module
|
152
|
+
# ----------------------------------------------------------------------------
|
153
|
+
|
154
|
+
NewOptionCreator = ForwardRef('NewOptionCreator')
|
155
|
+
|
156
|
+
class StaticFormOptionsDataSource(SubscriptableBaseModel):
|
157
|
+
"""
|
158
|
+
A Data Source for providing a static set of form options
|
159
|
+
|
160
|
+
:param List[FormOption] values: The list of values to return
|
161
|
+
:param NewOptionCreator new_option_creator: If provided, it means that values can be added to the datasource via the provided mechanism
|
162
|
+
:return: nothing
|
163
|
+
"""
|
164
|
+
values:List[FormOption] = Field(default_factory=list)
|
165
|
+
new_option_creator:NewOptionCreator = None
|
166
|
+
type:Literal["static"] = 'static'
|
167
|
+
|
168
|
+
class DynamicFormOptionsDataSource(SubscriptableBaseModel):
|
169
|
+
"""
|
170
|
+
A Data Source for providing a set of form options that load dynamically from the server
|
171
|
+
"""
|
172
|
+
source_function:Union[Callable[[SyncConfigurationParameters], List[FormOption]],str]
|
173
|
+
new_option_creator:NewOptionCreator = None
|
174
|
+
type:Literal["dynamic"] = 'dynamic'
|
175
|
+
|
176
|
+
@validator('source_function', always=True)
|
177
|
+
def function_name_convertor(cls, v, values) -> str:
|
178
|
+
return v.__name__ if isinstance(v,MethodType) else v
|
179
|
+
|
180
|
+
FormOptionsDataSourceBase = Annotated[Union[StaticFormOptionsDataSource,DynamicFormOptionsDataSource], Field(discriminator='type')]
|
181
|
+
|
182
|
+
class FormFieldWithDataSource(SubscriptableBaseModel):
|
183
|
+
"""
|
184
|
+
Denotes that the field uses a data source
|
185
|
+
"""
|
186
|
+
data_source:FormOptionsDataSourceBase
|
187
|
+
|
188
|
+
class FormRadioField(FormFieldWithDataSource,BaseModel):
|
189
|
+
"""
|
190
|
+
A field which presents a set of radio options
|
191
|
+
:param str name: The name of the form field. This value must be unique, and is used to retrieve the value from within the plugin code.
|
192
|
+
:param str label: The label for the form field
|
193
|
+
:param FormOptionsDataSourceBase data_source provides the values for the radio group
|
194
|
+
:param str default_value: The default value presented initially in the field
|
195
|
+
:param bool required: If True, means that the form cannot be submitted without a value present
|
196
|
+
:param str depends_on: The name of another form field. If provided, this form field will not be rendered until there is a value in the field it depends on.
|
197
|
+
:param str help_text: A longer description of what the field is used for. If provided, a help icon is shown and must be hovered over to display this text.
|
198
|
+
:param bool reload_on_change: If True, the entire form is reloaded after the value is changed. This is used to conditionally render fields based on values provided in others, but should be used only when strictly necessary.
|
199
|
+
:return: nothing
|
200
|
+
"""
|
201
|
+
name:str
|
202
|
+
label:str
|
203
|
+
default_value:str=None
|
204
|
+
required:bool=False
|
205
|
+
secret:bool=False
|
206
|
+
depends_on:str=None
|
207
|
+
help_text:str=""
|
208
|
+
reload_on_change:bool=False
|
209
|
+
type:Literal['radio'] = 'radio'
|
210
|
+
|
211
|
+
class FormDropdownField(FormFieldWithDataSource,BaseModel):
|
212
|
+
"""
|
213
|
+
A field which presents a dropdown list of options
|
214
|
+
|
215
|
+
"""
|
216
|
+
name:str
|
217
|
+
label:str
|
218
|
+
default_value:str=None
|
219
|
+
required:bool=False
|
220
|
+
secret:bool=False
|
221
|
+
depends_on:str=None
|
222
|
+
help_text:str=""
|
223
|
+
reload_on_change:bool=False
|
224
|
+
type:Literal['dropdown'] = 'dropdown'
|
225
|
+
|
226
|
+
multi_select:bool=False
|
227
|
+
|
228
|
+
FormFieldBase = Annotated[Union[FormInputField,FormTextAreaField,FormSshKeypair,FormRadioField,FormCheckboxField,FormSliderField,FormDropdownField], Field(discriminator='type')]
|
229
|
+
|
230
|
+
class ConfigurationFormBase(BaseModel,ABC):
|
231
|
+
"""
|
232
|
+
Defines a form for configuring a sync. Includes zero or more form fields.
|
233
|
+
|
234
|
+
:param List[FormFieldBase] fields: A list of fields to display to the user
|
235
|
+
:return: nothing
|
236
|
+
"""
|
237
|
+
fields:List[FormFieldBase]
|
238
|
+
|
239
|
+
class NewOptionCreator(SubscriptableBaseModel):
|
240
|
+
"""
|
241
|
+
Allows for options to be added to a datasource by the user.
|
242
|
+
It does this by presenting the user with a form, then building a FormOption from the provided values.
|
243
|
+
"""
|
244
|
+
creation_form_function:Union[Callable[[SyncConfigurationParameters], ConfigurationFormBase],str]
|
245
|
+
creation_complete_function:Union[Callable[[SyncConfigurationParameters], FormOption],str]
|
246
|
+
allow_create:bool = True
|
247
|
+
|
248
|
+
@validator('creation_form_function', always=True)
|
249
|
+
def function_name_convertor(cls, v, values) -> str:
|
250
|
+
return v.__name__ if isinstance(v,MethodType) else v
|
251
|
+
|
252
|
+
@validator('creation_complete_function', always=True)
|
253
|
+
def function_name_convertor_2(cls, v, values) -> str:
|
254
|
+
return v.__name__ if isinstance(v,MethodType) else v
|
255
|
+
|
256
|
+
StaticFormOptionsDataSource.update_forward_refs()
|
257
|
+
DynamicFormOptionsDataSource.update_forward_refs()
|
258
|
+
|
259
|
+
class FormFieldMappingSelector(FormFieldWithDataSource,BaseModel):
|
260
|
+
"""
|
261
|
+
Uses a visual column->field mapper, to allow the user to define how source columns map to app fields
|
262
|
+
|
263
|
+
:param FormOptionsDataSourceBase data_source: A data source which provides the app field options
|
264
|
+
:param str depends_on: Provide the name of another field to make it dependant, so that the mapper won't display until a value has been provided
|
265
|
+
:return: nothing
|
266
|
+
"""
|
267
|
+
mapper_type:Literal["field_mapping_selector"] = 'field_mapping_selector'
|
268
|
+
label:str="Field Mappings"
|
269
|
+
depends_on:str=None
|
270
|
+
|
271
|
+
|
272
|
+
Mapper = Annotated[Union[FormFieldMappingSelector,FormJinjaTemplate], Field(discriminator='mapper_type')]
|
273
|
+
|
274
|
+
class OutboundSyncConfigurationForm(ConfigurationFormBase):
|
275
|
+
"""
|
276
|
+
Defines a form for configuring an outbound sync.
|
277
|
+
Includes the zero or more form fields from the base class, and optionally a column->field mapper
|
278
|
+
to map Snowflake columns to app fields/payloads.
|
279
|
+
"""
|
280
|
+
mapper:Optional[Mapper] = None
|
281
|
+
|
282
|
+
class InboundSyncConfigurationForm(ConfigurationFormBase):
|
283
|
+
"""
|
284
|
+
Defines a form for configuring an inbound sync.
|
285
|
+
Includes the zero or more form fields from the base class, and then a means of displaying stream information.
|
286
|
+
"""
|
287
|
+
fields:List[FormFieldBase] = Field(default_factory=list)
|
288
|
+
stream_lister:StreamLister
|
289
|
+
|
290
|
+
class ConnectionMethod(SubscriptableBaseModel):
|
291
|
+
"""
|
292
|
+
Defines a method of connecting to an application.
|
293
|
+
:param str data_source: The name of the connection method, e.g. "OAuth", "API Key", "Credentials"
|
294
|
+
:param List[FormFieldBase] fields: A list of fields that are used to collect the connection information from the user.
|
295
|
+
:param List[str] network_addresses: A list of URLs that will be added to a network rule to permit outbound access from Snowflake
|
296
|
+
for the authentication step. Note that for OAuth Authorization flows, it is not necessary to provide the
|
297
|
+
initial URL that the user agent is directed to.
|
298
|
+
:param bool oauth: If True, the resulting form will indicate to the user that OAuth will be used.
|
299
|
+
In this scenario, the oauth_parameters function will be called before the connect function.
|
300
|
+
"""
|
301
|
+
name:str
|
302
|
+
fields:List[FormFieldBase]
|
303
|
+
network_addresses:List[str]
|
304
|
+
oauth:bool=False
|
305
|
+
|
306
|
+
|
@@ -0,0 +1,91 @@
|
|
1
|
+
"""
|
2
|
+
Custom logging functionality for Omnata
|
3
|
+
"""
|
4
|
+
import threading
|
5
|
+
import datetime
|
6
|
+
import logging
|
7
|
+
import logging.handlers
|
8
|
+
from typing import List
|
9
|
+
import pandas
|
10
|
+
from snowflake.snowpark import Session
|
11
|
+
|
12
|
+
class OmnataPluginLogHandler(logging.handlers.BufferingHandler):
|
13
|
+
"""
|
14
|
+
A logging handler to ship logs back into Omnata Snowflake tables. It uses a combination
|
15
|
+
of time and capacity to flush the buffer.
|
16
|
+
Additional information about the current sync and run is included, so that logs can be filtered easily.
|
17
|
+
"""
|
18
|
+
def __init__(self,session:Session,
|
19
|
+
sync_id:int,
|
20
|
+
sync_branch_id:int,
|
21
|
+
connection_id:int,
|
22
|
+
sync_run_id:int,
|
23
|
+
capacity=100,
|
24
|
+
duration=5,
|
25
|
+
):
|
26
|
+
logging.handlers.BufferingHandler.__init__(self,capacity=capacity)
|
27
|
+
self.session = session
|
28
|
+
self.duration = duration
|
29
|
+
self.timer = None
|
30
|
+
self.sync_id = sync_id
|
31
|
+
self.sync_branch_id = sync_branch_id
|
32
|
+
self.connection_id = connection_id
|
33
|
+
self.sync_run_id = sync_run_id
|
34
|
+
#formatter = logging.Formatter('%(message)s')
|
35
|
+
# add formatter to ch
|
36
|
+
#self.setFormatter(formatter)
|
37
|
+
self.init_timer()
|
38
|
+
|
39
|
+
def register(self,logging_level:str,additional_loggers:List[str] = None):
|
40
|
+
"""
|
41
|
+
Register the handler with the omnata_plugin namespace
|
42
|
+
"""
|
43
|
+
self.setLevel(logging_level)
|
44
|
+
logger = logging.getLogger('omnata_plugin')
|
45
|
+
logger.addHandler(self)
|
46
|
+
if additional_loggers is not None:
|
47
|
+
for additional_logger in additional_loggers:
|
48
|
+
logger = logging.getLogger(additional_logger)
|
49
|
+
logger.addHandler(self)
|
50
|
+
|
51
|
+
def unregister(self):
|
52
|
+
"""
|
53
|
+
Removes the handler
|
54
|
+
"""
|
55
|
+
logger = logging.getLogger('omnata_plugin')
|
56
|
+
logger.removeHandler(self)
|
57
|
+
|
58
|
+
def init_timer(self):
|
59
|
+
"""
|
60
|
+
Initialises the timer to trigger the flush after the duration is reached.
|
61
|
+
Will clear any existing timer.
|
62
|
+
"""
|
63
|
+
if self.timer is not None and self.timer.is_alive():
|
64
|
+
self.timer.cancel()
|
65
|
+
self.timer = threading.Timer(self.duration,self.flush)
|
66
|
+
|
67
|
+
def flush(self):
|
68
|
+
"""
|
69
|
+
Send the log records to Snowflake
|
70
|
+
"""
|
71
|
+
if len(self.buffer) > 0:
|
72
|
+
try:
|
73
|
+
results_df = pandas.DataFrame.from_dict([{
|
74
|
+
'SYNC_ID':self.sync_id,
|
75
|
+
'SYNC_BRANCH_ID':self.sync_branch_id,
|
76
|
+
'CONNECTION_ID':self.connection_id,
|
77
|
+
'SYNC_RUN_ID':self.sync_run_id,
|
78
|
+
'STREAM_NAME':data.__dict__['stream_name'] if 'stream_name' in data.__dict__ else None,
|
79
|
+
'LOG_LEVEL_NAME':data.levelname,
|
80
|
+
'LOG_LEVEL_NO':data.levelno,
|
81
|
+
'LOG_MESSAGE': data.getMessage(),
|
82
|
+
'LOG_STACK_TRACE': data.stack_info,
|
83
|
+
'EVENT_DATETIME': str(datetime.datetime.fromtimestamp(data.created))
|
84
|
+
} for data in self.buffer])
|
85
|
+
snowflake_df = self.session.create_dataframe(results_df)
|
86
|
+
snowflake_df.write.save_as_table(table_name='DATA.SYNC_RUN_LOG',mode='append',column_order='name')
|
87
|
+
except Exception:
|
88
|
+
self.handleError(None) # no particular record
|
89
|
+
self.buffer = []
|
90
|
+
# restart the timer, regardless of the reason for the flush
|
91
|
+
self.init_timer()
|