omnata-plugin-runtime 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,306 @@
1
+ # it's not the 1980s anymore
2
+ # pylint: disable=line-too-long,multiple-imports,logging-fstring-interpolation,too-many-arguments,no-self-argument
3
+ """
4
+ Contains form elements for Omnata plugin configuration
5
+ """
6
+ from __future__ import annotations
7
+ import sys
8
+ from typing import List,Callable, Literal, Union, ForwardRef, Optional
9
+ if tuple(sys.version_info[:2]) >= (3, 9):
10
+ # Python 3.9 and above
11
+ from typing import Annotated
12
+ else:
13
+ # Python 3.8 and below
14
+ from typing_extensions import Annotated
15
+ from abc import ABC
16
+ from types import MethodType
17
+ from pydantic import BaseModel,Field,validator # pylint: disable=no-name-in-module
18
+ from .configuration import SubscriptableBaseModel, StreamConfiguration, SyncConfigurationParameters, InboundSyncConfigurationParameters
19
+
20
+ class FormOption(SubscriptableBaseModel):
21
+ """
22
+ An option used by certain form forms (like Dropdowns).
23
+
24
+ :param str value: The value to set in the field if this option is selected
25
+ :param str label: The label to show in the list. This value is not stored.
26
+ :param dict metadata: An arbitrary dictionary to store with the value, which can be retrieved by the plugin
27
+ :param bool required: When populating field mapping options, this flag indicates that this field is mandatory
28
+ :param bool unique: When populating field mapping options, this flag indicates that this field requires a unique value (i.e. be mapped from a unique column)
29
+ :param bool default: Indicates that this option should be the default selected
30
+ :param bool disabled: Indicates that the option should appear in the list, but be unselectable
31
+ :param str data_type_icon: The data type icon to show next to the option (where applicable)
32
+ :return: nothing
33
+ """
34
+ value:str
35
+ label:str
36
+ metadata:dict = Field(default_factory=dict)
37
+ required:bool=False
38
+ unique:bool=False
39
+ default:bool=False
40
+ disabled:bool=False
41
+ data_type_icon:str='unknown'
42
+
43
+ class FormInputField(SubscriptableBaseModel):
44
+ """
45
+ An input field, which collects a single line of free-form text from the user and no metadata.
46
+ """
47
+ name:str
48
+ label:str
49
+ default_value:Union[str,bool]=''
50
+ required:bool=False
51
+ depends_on:str=None
52
+ help_text:str=""
53
+ reload_on_change:bool=False
54
+ secret:bool=False
55
+ type:Literal['input'] = 'input'
56
+
57
+ class FormTextAreaField(SubscriptableBaseModel):
58
+ """
59
+ A text area field, which collects multi-line free-form text from the user and no metadata.
60
+
61
+ :param str name: The name of the form field. This value must be unique, and is used to retrieve the value from within the plugin code.
62
+ :param str label: The label for the form field
63
+ :param str default_value: The default value presented initially in the field
64
+ :param bool required: If True, means that the form cannot be submitted without a value present
65
+ :param str depends_on: The name of another form field. If provided, this form field will not be rendered until there is a value in the field it depends on.
66
+ :param str help_text: A longer description of what the field is used for. If provided, a help icon is shown and must be hovered over to display this text.
67
+ :param bool secret: Indicates that the text entered must be masked in the browser, and stored/access securely
68
+ :param bool reload_on_change: If True, the entire form is reloaded after the value is changed. This is used to conditionally render fields based on values provided in others, but should be used only when strictly necessary.
69
+ :return: nothing
70
+ """
71
+ name:str
72
+ label:str
73
+ default_value:str=''
74
+ secret:bool=False
75
+ required:bool=False
76
+ depends_on:str=None
77
+ help_text:str=""
78
+ reload_on_change:bool=False
79
+ type:Literal['textarea'] = 'textarea'
80
+
81
+ variables:bool=False
82
+
83
+ class FormSshKeypair(SubscriptableBaseModel):
84
+ """
85
+ An SSH Keypair field, which generates public and private keys for asymmetric cryptography.
86
+ """
87
+ name:str
88
+ label:str
89
+ default_value:str=None
90
+ required:bool=False
91
+ depends_on:str=None
92
+ help_text:str=""
93
+
94
+ type:Literal['ssh_keypair'] = 'ssh_keypair'
95
+ secret:bool=True
96
+
97
+ class FormCheckboxField(SubscriptableBaseModel):
98
+ """
99
+ A field which presents a checkbox
100
+ """
101
+ name:str
102
+ label:str
103
+ default_value:bool=False
104
+ required:bool=False
105
+ secret:bool=False
106
+ depends_on:str=None
107
+ help_text:str=""
108
+ reload_on_change:bool=False
109
+ type:Literal['checkbox'] = 'checkbox'
110
+
111
+ class FormSliderField(SubscriptableBaseModel):
112
+ """
113
+ A field which presents a slider
114
+ """
115
+ name:str
116
+ label:str
117
+ default_value:str=None
118
+ secret:bool=False
119
+ required:bool=False
120
+ depends_on:str=None
121
+ help_text:str=""
122
+ reload_on_change:bool=False
123
+ type:Literal['slider'] = 'slider'
124
+
125
+ min_value:int=0
126
+ max_value:int=100
127
+ step_size:int=1
128
+
129
+ class StreamLister(SubscriptableBaseModel):
130
+ """
131
+ A class for managing the listing of Streams. Can depend on other form fields to delay rendering
132
+ """
133
+ source_function:Union[Callable[[InboundSyncConfigurationParameters], List[StreamConfiguration]],str]
134
+ label:str="Select Objects"
135
+ depends_on:str=None
136
+
137
+ @validator('source_function', always=True)
138
+ def function_name_convertor(cls, v, values) -> str:
139
+ return v.__name__ if isinstance(v,MethodType) else v
140
+
141
+ class FormJinjaTemplate(SubscriptableBaseModel):
142
+ """
143
+ Uses text area to allow the user to create a template, which can include column values from the source
144
+ """
145
+ mapper_type:Literal["jinja_template"] = 'jinja_template'
146
+ label:str="Jinja Template"
147
+ label:str=None
148
+ depends_on:str=None
149
+
150
+ # ----------------------------------------------------------------------------
151
+ # Everything above here has no dependencies on other BaseModels in this module
152
+ # ----------------------------------------------------------------------------
153
+
154
+ NewOptionCreator = ForwardRef('NewOptionCreator')
155
+
156
+ class StaticFormOptionsDataSource(SubscriptableBaseModel):
157
+ """
158
+ A Data Source for providing a static set of form options
159
+
160
+ :param List[FormOption] values: The list of values to return
161
+ :param NewOptionCreator new_option_creator: If provided, it means that values can be added to the datasource via the provided mechanism
162
+ :return: nothing
163
+ """
164
+ values:List[FormOption] = Field(default_factory=list)
165
+ new_option_creator:NewOptionCreator = None
166
+ type:Literal["static"] = 'static'
167
+
168
+ class DynamicFormOptionsDataSource(SubscriptableBaseModel):
169
+ """
170
+ A Data Source for providing a set of form options that load dynamically from the server
171
+ """
172
+ source_function:Union[Callable[[SyncConfigurationParameters], List[FormOption]],str]
173
+ new_option_creator:NewOptionCreator = None
174
+ type:Literal["dynamic"] = 'dynamic'
175
+
176
+ @validator('source_function', always=True)
177
+ def function_name_convertor(cls, v, values) -> str:
178
+ return v.__name__ if isinstance(v,MethodType) else v
179
+
180
+ FormOptionsDataSourceBase = Annotated[Union[StaticFormOptionsDataSource,DynamicFormOptionsDataSource], Field(discriminator='type')]
181
+
182
+ class FormFieldWithDataSource(SubscriptableBaseModel):
183
+ """
184
+ Denotes that the field uses a data source
185
+ """
186
+ data_source:FormOptionsDataSourceBase
187
+
188
+ class FormRadioField(FormFieldWithDataSource,BaseModel):
189
+ """
190
+ A field which presents a set of radio options
191
+ :param str name: The name of the form field. This value must be unique, and is used to retrieve the value from within the plugin code.
192
+ :param str label: The label for the form field
193
+ :param FormOptionsDataSourceBase data_source provides the values for the radio group
194
+ :param str default_value: The default value presented initially in the field
195
+ :param bool required: If True, means that the form cannot be submitted without a value present
196
+ :param str depends_on: The name of another form field. If provided, this form field will not be rendered until there is a value in the field it depends on.
197
+ :param str help_text: A longer description of what the field is used for. If provided, a help icon is shown and must be hovered over to display this text.
198
+ :param bool reload_on_change: If True, the entire form is reloaded after the value is changed. This is used to conditionally render fields based on values provided in others, but should be used only when strictly necessary.
199
+ :return: nothing
200
+ """
201
+ name:str
202
+ label:str
203
+ default_value:str=None
204
+ required:bool=False
205
+ secret:bool=False
206
+ depends_on:str=None
207
+ help_text:str=""
208
+ reload_on_change:bool=False
209
+ type:Literal['radio'] = 'radio'
210
+
211
+ class FormDropdownField(FormFieldWithDataSource,BaseModel):
212
+ """
213
+ A field which presents a dropdown list of options
214
+
215
+ """
216
+ name:str
217
+ label:str
218
+ default_value:str=None
219
+ required:bool=False
220
+ secret:bool=False
221
+ depends_on:str=None
222
+ help_text:str=""
223
+ reload_on_change:bool=False
224
+ type:Literal['dropdown'] = 'dropdown'
225
+
226
+ multi_select:bool=False
227
+
228
+ FormFieldBase = Annotated[Union[FormInputField,FormTextAreaField,FormSshKeypair,FormRadioField,FormCheckboxField,FormSliderField,FormDropdownField], Field(discriminator='type')]
229
+
230
+ class ConfigurationFormBase(BaseModel,ABC):
231
+ """
232
+ Defines a form for configuring a sync. Includes zero or more form fields.
233
+
234
+ :param List[FormFieldBase] fields: A list of fields to display to the user
235
+ :return: nothing
236
+ """
237
+ fields:List[FormFieldBase]
238
+
239
+ class NewOptionCreator(SubscriptableBaseModel):
240
+ """
241
+ Allows for options to be added to a datasource by the user.
242
+ It does this by presenting the user with a form, then building a FormOption from the provided values.
243
+ """
244
+ creation_form_function:Union[Callable[[SyncConfigurationParameters], ConfigurationFormBase],str]
245
+ creation_complete_function:Union[Callable[[SyncConfigurationParameters], FormOption],str]
246
+ allow_create:bool = True
247
+
248
+ @validator('creation_form_function', always=True)
249
+ def function_name_convertor(cls, v, values) -> str:
250
+ return v.__name__ if isinstance(v,MethodType) else v
251
+
252
+ @validator('creation_complete_function', always=True)
253
+ def function_name_convertor_2(cls, v, values) -> str:
254
+ return v.__name__ if isinstance(v,MethodType) else v
255
+
256
+ StaticFormOptionsDataSource.update_forward_refs()
257
+ DynamicFormOptionsDataSource.update_forward_refs()
258
+
259
+ class FormFieldMappingSelector(FormFieldWithDataSource,BaseModel):
260
+ """
261
+ Uses a visual column->field mapper, to allow the user to define how source columns map to app fields
262
+
263
+ :param FormOptionsDataSourceBase data_source: A data source which provides the app field options
264
+ :param str depends_on: Provide the name of another field to make it dependant, so that the mapper won't display until a value has been provided
265
+ :return: nothing
266
+ """
267
+ mapper_type:Literal["field_mapping_selector"] = 'field_mapping_selector'
268
+ label:str="Field Mappings"
269
+ depends_on:str=None
270
+
271
+
272
+ Mapper = Annotated[Union[FormFieldMappingSelector,FormJinjaTemplate], Field(discriminator='mapper_type')]
273
+
274
+ class OutboundSyncConfigurationForm(ConfigurationFormBase):
275
+ """
276
+ Defines a form for configuring an outbound sync.
277
+ Includes the zero or more form fields from the base class, and optionally a column->field mapper
278
+ to map Snowflake columns to app fields/payloads.
279
+ """
280
+ mapper:Optional[Mapper] = None
281
+
282
+ class InboundSyncConfigurationForm(ConfigurationFormBase):
283
+ """
284
+ Defines a form for configuring an inbound sync.
285
+ Includes the zero or more form fields from the base class, and then a means of displaying stream information.
286
+ """
287
+ fields:List[FormFieldBase] = Field(default_factory=list)
288
+ stream_lister:StreamLister
289
+
290
+ class ConnectionMethod(SubscriptableBaseModel):
291
+ """
292
+ Defines a method of connecting to an application.
293
+ :param str data_source: The name of the connection method, e.g. "OAuth", "API Key", "Credentials"
294
+ :param List[FormFieldBase] fields: A list of fields that are used to collect the connection information from the user.
295
+ :param List[str] network_addresses: A list of URLs that will be added to a network rule to permit outbound access from Snowflake
296
+ for the authentication step. Note that for OAuth Authorization flows, it is not necessary to provide the
297
+ initial URL that the user agent is directed to.
298
+ :param bool oauth: If True, the resulting form will indicate to the user that OAuth will be used.
299
+ In this scenario, the oauth_parameters function will be called before the connect function.
300
+ """
301
+ name:str
302
+ fields:List[FormFieldBase]
303
+ network_addresses:List[str]
304
+ oauth:bool=False
305
+
306
+
@@ -0,0 +1,91 @@
1
+ """
2
+ Custom logging functionality for Omnata
3
+ """
4
+ import threading
5
+ import datetime
6
+ import logging
7
+ import logging.handlers
8
+ from typing import List
9
+ import pandas
10
+ from snowflake.snowpark import Session
11
+
12
+ class OmnataPluginLogHandler(logging.handlers.BufferingHandler):
13
+ """
14
+ A logging handler to ship logs back into Omnata Snowflake tables. It uses a combination
15
+ of time and capacity to flush the buffer.
16
+ Additional information about the current sync and run is included, so that logs can be filtered easily.
17
+ """
18
+ def __init__(self,session:Session,
19
+ sync_id:int,
20
+ sync_branch_id:int,
21
+ connection_id:int,
22
+ sync_run_id:int,
23
+ capacity=100,
24
+ duration=5,
25
+ ):
26
+ logging.handlers.BufferingHandler.__init__(self,capacity=capacity)
27
+ self.session = session
28
+ self.duration = duration
29
+ self.timer = None
30
+ self.sync_id = sync_id
31
+ self.sync_branch_id = sync_branch_id
32
+ self.connection_id = connection_id
33
+ self.sync_run_id = sync_run_id
34
+ #formatter = logging.Formatter('%(message)s')
35
+ # add formatter to ch
36
+ #self.setFormatter(formatter)
37
+ self.init_timer()
38
+
39
+ def register(self,logging_level:str,additional_loggers:List[str] = None):
40
+ """
41
+ Register the handler with the omnata_plugin namespace
42
+ """
43
+ self.setLevel(logging_level)
44
+ logger = logging.getLogger('omnata_plugin')
45
+ logger.addHandler(self)
46
+ if additional_loggers is not None:
47
+ for additional_logger in additional_loggers:
48
+ logger = logging.getLogger(additional_logger)
49
+ logger.addHandler(self)
50
+
51
+ def unregister(self):
52
+ """
53
+ Removes the handler
54
+ """
55
+ logger = logging.getLogger('omnata_plugin')
56
+ logger.removeHandler(self)
57
+
58
+ def init_timer(self):
59
+ """
60
+ Initialises the timer to trigger the flush after the duration is reached.
61
+ Will clear any existing timer.
62
+ """
63
+ if self.timer is not None and self.timer.is_alive():
64
+ self.timer.cancel()
65
+ self.timer = threading.Timer(self.duration,self.flush)
66
+
67
+ def flush(self):
68
+ """
69
+ Send the log records to Snowflake
70
+ """
71
+ if len(self.buffer) > 0:
72
+ try:
73
+ results_df = pandas.DataFrame.from_dict([{
74
+ 'SYNC_ID':self.sync_id,
75
+ 'SYNC_BRANCH_ID':self.sync_branch_id,
76
+ 'CONNECTION_ID':self.connection_id,
77
+ 'SYNC_RUN_ID':self.sync_run_id,
78
+ 'STREAM_NAME':data.__dict__['stream_name'] if 'stream_name' in data.__dict__ else None,
79
+ 'LOG_LEVEL_NAME':data.levelname,
80
+ 'LOG_LEVEL_NO':data.levelno,
81
+ 'LOG_MESSAGE': data.getMessage(),
82
+ 'LOG_STACK_TRACE': data.stack_info,
83
+ 'EVENT_DATETIME': str(datetime.datetime.fromtimestamp(data.created))
84
+ } for data in self.buffer])
85
+ snowflake_df = self.session.create_dataframe(results_df)
86
+ snowflake_df.write.save_as_table(table_name='DATA.SYNC_RUN_LOG',mode='append',column_order='name')
87
+ except Exception:
88
+ self.handleError(None) # no particular record
89
+ self.buffer = []
90
+ # restart the timer, regardless of the reason for the flush
91
+ self.init_timer()