omnata-plugin-runtime 0.8.0a187__tar.gz → 0.8.0a189__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/PKG-INFO +1 -1
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/pyproject.toml +1 -1
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/configuration.py +3 -1
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/plugin_entrypoints.py +152 -127
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/LICENSE +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/README.md +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/__init__.py +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/api.py +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/forms.py +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/logging.py +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/omnata_plugin.py +0 -0
- {omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/rate_limiting.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "omnata-plugin-runtime"
|
|
3
|
-
version = "0.8.0-
|
|
3
|
+
version = "0.8.0-a189"
|
|
4
4
|
description = "Classes and common runtime components for building and running Omnata Plugins"
|
|
5
5
|
authors = ["James Weakley <james.weakley@omnata.com>"]
|
|
6
6
|
readme = "README.md"
|
|
@@ -19,7 +19,9 @@ if tuple(sys.version_info[:2]) >= (3, 9):
|
|
|
19
19
|
else:
|
|
20
20
|
# Python 3.8 and below
|
|
21
21
|
from typing_extensions import Annotated
|
|
22
|
+
from opentelemetry import trace
|
|
22
23
|
|
|
24
|
+
tracer = trace.get_tracer(__name__)
|
|
23
25
|
|
|
24
26
|
class MapperType(str, Enum):
|
|
25
27
|
FIELD_MAPPING_SELECTOR = "field_mapping_selector"
|
|
@@ -871,7 +873,7 @@ InboundSyncStreamsConfiguration.model_rebuild()
|
|
|
871
873
|
StoredFieldMappings.model_rebuild()
|
|
872
874
|
OutboundSyncConfigurationParameters.model_rebuild()
|
|
873
875
|
|
|
874
|
-
|
|
876
|
+
@tracer.start_as_current_span("get_secrets")
|
|
875
877
|
def get_secrets(oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
|
|
876
878
|
) -> Dict[str, StoredConfigurationValue]:
|
|
877
879
|
connection_secrets = {}
|
|
@@ -36,9 +36,10 @@ from .omnata_plugin import (
|
|
|
36
36
|
)
|
|
37
37
|
from pydantic import TypeAdapter
|
|
38
38
|
from .rate_limiting import ApiLimits, RateLimitState
|
|
39
|
+
from opentelemetry import trace
|
|
39
40
|
|
|
40
41
|
IMPORT_DIRECTORY_NAME = "snowflake_import_directory"
|
|
41
|
-
|
|
42
|
+
tracer = trace.get_tracer(__name__)
|
|
42
43
|
|
|
43
44
|
class PluginEntrypoint:
|
|
44
45
|
"""
|
|
@@ -50,46 +51,39 @@ class PluginEntrypoint:
|
|
|
50
51
|
self, plugin_fqn: str, session: Session, module_name: str, class_name: str
|
|
51
52
|
):
|
|
52
53
|
logger.info(f"Initialising plugin entrypoint for {plugin_fqn}")
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
54
|
+
with tracer.start_as_current_span("plugin_initialization") as span:
|
|
55
|
+
self._session = session
|
|
56
|
+
import_dir = sys._xoptions[IMPORT_DIRECTORY_NAME]
|
|
57
|
+
span.add_event("Adding plugin zip to path")
|
|
58
|
+
sys.path.append(os.path.join(import_dir, "app.zip"))
|
|
59
|
+
span.add_event("Importing plugin module")
|
|
60
|
+
module = importlib.import_module(module_name)
|
|
61
|
+
class_obj = getattr(module, class_name)
|
|
62
|
+
self._plugin_instance: OmnataPlugin = class_obj()
|
|
63
|
+
self._plugin_instance._session = session # pylint: disable=protected-access
|
|
64
|
+
# logging defaults
|
|
65
|
+
snowflake_logger = logging.getLogger("snowflake")
|
|
66
|
+
snowflake_logger.setLevel(logging.WARN) # we don't want snowflake queries being logged by default
|
|
67
|
+
# the sync engine can tell the plugin to override log level via a session variable
|
|
68
|
+
if session is not None:
|
|
69
|
+
try:
|
|
70
|
+
span.add_event("Checking log level overrides")
|
|
71
|
+
v = session.sql("select getvariable('LOG_LEVEL_OVERRIDES')").collect()
|
|
72
|
+
result = v[0][0]
|
|
73
|
+
if result is not None:
|
|
74
|
+
log_level_overrides:Dict[str,str] = json.loads(result)
|
|
75
|
+
span.add_event("Applying log level overrides",log_level_overrides)
|
|
76
|
+
for logger_name,level in log_level_overrides.items():
|
|
77
|
+
logger_override = logging.getLogger(logger_name)
|
|
78
|
+
logger_override.setLevel(level)
|
|
79
|
+
logger_override.propagate = False
|
|
80
|
+
for handler in logger_override.handlers:
|
|
81
|
+
handler.setLevel(level)
|
|
82
|
+
except Exception as e:
|
|
83
|
+
logger.error(f"Error setting log level overrides: {str(e)}")
|
|
78
84
|
|
|
79
85
|
|
|
80
86
|
def sync(self, sync_request: Dict):
|
|
81
|
-
logger.info("Entered sync method")
|
|
82
|
-
request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
|
|
83
|
-
connection_secrets = get_secrets(
|
|
84
|
-
request.oauth_secret_name, request.other_secrets_name
|
|
85
|
-
)
|
|
86
|
-
omnata_log_handler = OmnataPluginLogHandler(
|
|
87
|
-
session=self._session,
|
|
88
|
-
sync_id=request.sync_id,
|
|
89
|
-
sync_branch_id=request.sync_branch_id,
|
|
90
|
-
connection_id=request.connection_id,
|
|
91
|
-
sync_run_id=request.run_id,
|
|
92
|
-
)
|
|
93
87
|
logger.add_extra('omnata.operation', 'sync')
|
|
94
88
|
logger.add_extra('omnata.sync.id', request.sync_id)
|
|
95
89
|
logger.add_extra('omnata.sync.direction', request.sync_direction)
|
|
@@ -97,50 +91,67 @@ class PluginEntrypoint:
|
|
|
97
91
|
logger.add_extra('omnata.sync.run_id', request.run_id)
|
|
98
92
|
logger.add_extra('omnata.sync_branch.id', request.sync_branch_id)
|
|
99
93
|
logger.add_extra('omnata.sync_branch.name', request.sync_branch_name)
|
|
94
|
+
logger.info("Entered sync method")
|
|
95
|
+
with tracer.start_as_current_span("initialization") as span:
|
|
96
|
+
span.add_event("Fetching secrets")
|
|
100
97
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
98
|
+
request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
|
|
99
|
+
connection_secrets = get_secrets(
|
|
100
|
+
request.oauth_secret_name, request.other_secrets_name
|
|
101
|
+
)
|
|
102
|
+
span.add_event("Configuring log handler")
|
|
103
|
+
omnata_log_handler = OmnataPluginLogHandler(
|
|
104
|
+
session=self._session,
|
|
105
|
+
sync_id=request.sync_id,
|
|
106
|
+
sync_branch_id=request.sync_branch_id,
|
|
107
|
+
connection_id=request.connection_id,
|
|
108
|
+
sync_run_id=request.run_id,
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
omnata_log_handler.register(
|
|
112
|
+
request.logging_level, self._plugin_instance.additional_loggers()
|
|
113
|
+
)
|
|
114
|
+
# construct some connection parameters for the purpose of getting the api limits
|
|
115
|
+
connection_parameters = ConnectionConfigurationParameters(
|
|
116
|
+
connection_method=request.connection_method,
|
|
117
|
+
connectivity_option=request.connectivity_option,
|
|
118
|
+
connection_parameters=request.connection_parameters,
|
|
119
|
+
connection_secrets=connection_secrets
|
|
120
|
+
)
|
|
121
|
+
if request.oauth_secret_name is not None:
|
|
122
|
+
connection_parameters.access_token_secret_name = request.oauth_secret_name
|
|
123
|
+
span.add_event("Configuring API Limits")
|
|
124
|
+
all_api_limits = self._plugin_instance.api_limits(connection_parameters)
|
|
125
|
+
logger.info(
|
|
126
|
+
f"Default API limits: {json.dumps(to_jsonable_python(all_api_limits))}"
|
|
127
|
+
)
|
|
128
|
+
all_api_limits_by_category = {
|
|
129
|
+
api_limit.endpoint_category: api_limit for api_limit in all_api_limits
|
|
126
130
|
}
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
131
|
+
all_api_limits_by_category.update(
|
|
132
|
+
{
|
|
133
|
+
k: v
|
|
134
|
+
for k, v in [
|
|
135
|
+
(x.endpoint_category, x) for x in request.api_limit_overrides
|
|
136
|
+
]
|
|
137
|
+
}
|
|
138
|
+
)
|
|
139
|
+
api_limits = list(all_api_limits_by_category.values())
|
|
140
|
+
return_dict = {}
|
|
141
|
+
logger.info(
|
|
142
|
+
f"Rate limits state: {json.dumps(to_jsonable_python(request.rate_limits_state))}"
|
|
143
|
+
)
|
|
144
|
+
(rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(request.rate_limits_state,request.sync_id, request.sync_branch_name)
|
|
145
|
+
# if any endpoint categories have no state, give them an empty state
|
|
146
|
+
for api_limit in api_limits:
|
|
147
|
+
if api_limit.endpoint_category not in rate_limit_state_all:
|
|
148
|
+
rate_limit_state_all[api_limit.endpoint_category] = RateLimitState(
|
|
149
|
+
wait_until=None, previous_request_timestamps=[]
|
|
150
|
+
)
|
|
151
|
+
if api_limit.endpoint_category not in rate_limit_state_this_branch:
|
|
152
|
+
rate_limit_state_this_branch[api_limit.endpoint_category] = RateLimitState(
|
|
153
|
+
wait_until=None, previous_request_timestamps=[]
|
|
154
|
+
)
|
|
144
155
|
|
|
145
156
|
if request.sync_direction == "outbound":
|
|
146
157
|
parameters = OutboundSyncConfigurationParameters(
|
|
@@ -174,11 +185,13 @@ class PluginEntrypoint:
|
|
|
174
185
|
)
|
|
175
186
|
try:
|
|
176
187
|
self._plugin_instance._configuration_parameters = parameters
|
|
177
|
-
with
|
|
178
|
-
|
|
188
|
+
with tracer.start_as_current_span("invoke_plugin") as span:
|
|
189
|
+
with HttpRateLimiting(outbound_sync_request, parameters):
|
|
190
|
+
self._plugin_instance.sync_outbound(parameters, outbound_sync_request)
|
|
179
191
|
if self._plugin_instance.disable_background_workers is False:
|
|
180
|
-
|
|
181
|
-
|
|
192
|
+
with tracer.start_as_current_span("results_finalization") as span:
|
|
193
|
+
outbound_sync_request.apply_results_queue()
|
|
194
|
+
outbound_sync_request.apply_rate_limit_state()
|
|
182
195
|
if outbound_sync_request.deadline_reached:
|
|
183
196
|
# if we actually hit the deadline, this is flagged by the cancellation checking worker and the cancellation
|
|
184
197
|
# token is set. We throw it here as an error since that's currently how it flows back to the engine with a DELAYED state
|
|
@@ -232,19 +245,21 @@ class PluginEntrypoint:
|
|
|
232
245
|
inbound_sync_request.update_activity("Invoking plugin")
|
|
233
246
|
logger.info(f"inbound sync request: {inbound_sync_request}")
|
|
234
247
|
# plugin_instance._inbound_sync_request = outbound_sync_request
|
|
235
|
-
with
|
|
236
|
-
|
|
248
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
249
|
+
with HttpRateLimiting(inbound_sync_request, parameters):
|
|
250
|
+
self._plugin_instance.sync_inbound(parameters, inbound_sync_request)
|
|
237
251
|
logger.info("Finished invoking plugin")
|
|
238
252
|
if self._plugin_instance.disable_background_workers is False:
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
253
|
+
with tracer.start_as_current_span("results_finalization") as span:
|
|
254
|
+
inbound_sync_request.update_activity("Staging remaining records")
|
|
255
|
+
logger.info("Calling apply_results_queue")
|
|
256
|
+
inbound_sync_request.apply_results_queue()
|
|
257
|
+
try:
|
|
258
|
+
# this is not critical, we wouldn't fail the sync over rate limit usage capture
|
|
259
|
+
logger.info("Calling apply_rate_limit_state")
|
|
260
|
+
inbound_sync_request.apply_rate_limit_state()
|
|
261
|
+
except Exception as e:
|
|
262
|
+
logger.error(f"Error applying rate limit state: {str(e)}")
|
|
248
263
|
# here we used to do a final inbound_sync_request.apply_progress_updates(ignore_errors=False)
|
|
249
264
|
# but it was erroring too much since there was usually a lot of DDL activity on the Snowflake side
|
|
250
265
|
# so instead, we'll provide a final progress update via a return value from the proc
|
|
@@ -288,6 +303,8 @@ class PluginEntrypoint:
|
|
|
288
303
|
sync_parameters: Dict,
|
|
289
304
|
current_form_parameters: Optional[Dict],
|
|
290
305
|
):
|
|
306
|
+
if function_name is None:
|
|
307
|
+
function_name = f"{sync_direction}_configuration_form"
|
|
291
308
|
logger.add_extra('omnata.operation', 'configuration_form')
|
|
292
309
|
logger.add_extra('omnata.connection.connectivity_option', connectivity_option)
|
|
293
310
|
logger.add_extra('omnata.connection.connection_method', connection_method)
|
|
@@ -333,9 +350,10 @@ class PluginEntrypoint:
|
|
|
333
350
|
parameters.access_token_secret_name = oauth_secret_name
|
|
334
351
|
the_function = getattr(
|
|
335
352
|
self._plugin_instance,
|
|
336
|
-
function_name
|
|
353
|
+
function_name
|
|
337
354
|
)
|
|
338
|
-
|
|
355
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
356
|
+
script_result = the_function(parameters)
|
|
339
357
|
if isinstance(script_result, BaseModel):
|
|
340
358
|
script_result = script_result.model_dump()
|
|
341
359
|
elif isinstance(script_result, List):
|
|
@@ -377,8 +395,8 @@ class PluginEntrypoint:
|
|
|
377
395
|
)
|
|
378
396
|
if oauth_secret_name is not None:
|
|
379
397
|
parameters.access_token_secret_name = oauth_secret_name
|
|
380
|
-
|
|
381
|
-
|
|
398
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
399
|
+
script_result = self._plugin_instance.inbound_stream_list(parameters)
|
|
382
400
|
if isinstance(script_result, BaseModel):
|
|
383
401
|
script_result = script_result.model_dump()
|
|
384
402
|
elif isinstance(script_result, List):
|
|
@@ -412,19 +430,21 @@ class PluginEntrypoint:
|
|
|
412
430
|
logger.add_extra('omnata.connection.connectivity_option', connectivity_option)
|
|
413
431
|
connectivity_option = TypeAdapter(ConnectivityOption).validate_python(connectivity_option)
|
|
414
432
|
logger.info("Entered connection_form method")
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
433
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
434
|
+
if self._plugin_instance.connection_form.__code__.co_argcount==1:
|
|
435
|
+
form: List[ConnectionMethod] = self._plugin_instance.connection_form()
|
|
436
|
+
else:
|
|
437
|
+
form: List[ConnectionMethod] = self._plugin_instance.connection_form(connectivity_option)
|
|
419
438
|
return [f.model_dump() for f in form]
|
|
420
439
|
|
|
421
440
|
def create_billing_events(self, session, event_request: Dict):
|
|
422
441
|
logger.add_extra('omnata.operation', 'create_billing_events')
|
|
423
442
|
logger.info("Entered create_billing_events method")
|
|
424
443
|
request = TypeAdapter(BillingEventRequest).validate_python(event_request)
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
444
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
445
|
+
events: List[SnowflakeBillingEvent] = self._plugin_instance.create_billing_events(
|
|
446
|
+
request
|
|
447
|
+
)
|
|
428
448
|
# create each billing event, waiting a second between each one
|
|
429
449
|
first_time = True
|
|
430
450
|
for billing_event in events:
|
|
@@ -511,31 +531,35 @@ class PluginEntrypoint:
|
|
|
511
531
|
)
|
|
512
532
|
if oauth_secret_name is not None:
|
|
513
533
|
parameters.access_token_secret_name = oauth_secret_name
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
534
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
535
|
+
connect_response = self._plugin_instance.connect(
|
|
536
|
+
parameters=parameters
|
|
537
|
+
)
|
|
517
538
|
# the connect method can also return more network addresses. If so, we need to update the
|
|
518
539
|
# network rule associated with the external access integration
|
|
519
540
|
if connect_response is None:
|
|
520
541
|
raise ValueError("Plugin did not return a ConnectResponse object from the connect method")
|
|
521
542
|
if connect_response.network_addresses is not None:
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
f"
|
|
538
|
-
|
|
543
|
+
with tracer.start_as_current_span("network_rule_update") as network_rule_update_span:
|
|
544
|
+
network_rule_update_span.add_event("Retrieving existing network rule")
|
|
545
|
+
existing_rule_result = self._session.sql(
|
|
546
|
+
f"desc network rule {network_rule_name}"
|
|
547
|
+
).collect()
|
|
548
|
+
rule_values: List[str] = existing_rule_result[0].value_list.split(",")
|
|
549
|
+
rule_values = [r for r in rule_values if r != '']
|
|
550
|
+
logger.info(f"Existing rules for {network_rule_name}: {rule_values}")
|
|
551
|
+
for network_address in connect_response.network_addresses:
|
|
552
|
+
if network_address not in rule_values:
|
|
553
|
+
rule_values.append(network_address)
|
|
554
|
+
#if len(rule_values)==0:
|
|
555
|
+
# logger.info("No network addresses for plugin, adding localhost")
|
|
556
|
+
# rule_values.append("https://localhost")
|
|
557
|
+
logger.info(f"New rules for {network_rule_name}: {rule_values}")
|
|
558
|
+
rule_values_string = ",".join([f"'{value}'" for value in rule_values])
|
|
559
|
+
network_rule_update_span.add_event("Updating network rule")
|
|
560
|
+
self._session.sql(
|
|
561
|
+
f"alter network rule {network_rule_name} set value_list = ({rule_values_string})"
|
|
562
|
+
).collect()
|
|
539
563
|
|
|
540
564
|
return connect_response.model_dump()
|
|
541
565
|
|
|
@@ -562,7 +586,8 @@ class PluginEntrypoint:
|
|
|
562
586
|
)
|
|
563
587
|
if oauth_secret_name is not None:
|
|
564
588
|
connection_parameters.access_token_secret_name = oauth_secret_name
|
|
565
|
-
|
|
589
|
+
with tracer.start_as_current_span("invoke_plugin"):
|
|
590
|
+
response: List[ApiLimits] = self._plugin_instance.api_limits(connection_parameters)
|
|
566
591
|
return [api_limit.model_dump() for api_limit in response]
|
|
567
592
|
|
|
568
593
|
def outbound_record_validator(
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{omnata_plugin_runtime-0.8.0a187 → omnata_plugin_runtime-0.8.0a189}/src/omnata_plugin_runtime/api.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|