omnata-plugin-runtime 0.5.5a127__tar.gz → 0.5.6__tar.gz
Sign up to get free protection for your applications and to get access to all the features.
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/PKG-INFO +2 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/pyproject.toml +2 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/configuration.py +31 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/omnata_plugin.py +314 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/plugin_entrypoints.py +7 -31
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/LICENSE +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/README.md +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/__init__.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/api.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/forms.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/logging.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/rate_limiting.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: omnata-plugin-runtime
|
3
|
-
Version: 0.5.
|
3
|
+
Version: 0.5.6
|
4
4
|
Summary: Classes and common runtime components for building and running Omnata Plugins
|
5
5
|
Author: James Weakley
|
6
6
|
Author-email: james.weakley@omnata.com
|
@@ -22,6 +22,7 @@ Requires-Dist: numpy (<=1.26.4)
|
|
22
22
|
Requires-Dist: packaging (<=24.1)
|
23
23
|
Requires-Dist: pandas (<=2.2.2)
|
24
24
|
Requires-Dist: platformdirs (<=3.10.0)
|
25
|
+
Requires-Dist: pyarrow (<=16.1.0)
|
25
26
|
Requires-Dist: pycparser (<=2.21)
|
26
27
|
Requires-Dist: pydantic (>=2,<=2.5.3)
|
27
28
|
Requires-Dist: pydantic-core (<=2.14.6)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "omnata-plugin-runtime"
|
3
|
-
version = "0.5.
|
3
|
+
version = "0.5.6"
|
4
4
|
description = "Classes and common runtime components for building and running Omnata Plugins"
|
5
5
|
authors = ["James Weakley <james.weakley@omnata.com>"]
|
6
6
|
readme = "README.md"
|
@@ -37,6 +37,7 @@ urllib3 = "<=2.2.2" # latest version available on Snowflake Anaconda
|
|
37
37
|
wheel = "<=0.43.0" # latest version available on Snowflake Anaconda
|
38
38
|
pyyaml = "<=6.0.1" # latest version available on Snowflake Anaconda
|
39
39
|
cffi = "<=1.16.0" # latest version available on Snowflake Anaconda
|
40
|
+
pyarrow = "<=16.1.0" # latest version available on Snowflake Anaconda
|
40
41
|
|
41
42
|
[tool.poetry.dev-dependencies]
|
42
43
|
pytest = "^6.2.4"
|
@@ -3,12 +3,14 @@ Omnata Plugin Runtime configuration objects.
|
|
3
3
|
Includes data container classes related to plugin configuration.
|
4
4
|
"""
|
5
5
|
from __future__ import annotations
|
6
|
+
import json
|
6
7
|
import sys
|
8
|
+
import logging
|
7
9
|
from typing import Any, List, Dict, Literal, Union, Optional
|
8
10
|
from enum import Enum
|
9
11
|
|
10
12
|
from abc import ABC
|
11
|
-
from pydantic import BaseModel, Field, PrivateAttr, SerializationInfo, field_validator, model_serializer, validator # pylint: disable=no-name-in-module
|
13
|
+
from pydantic import BaseModel, Field, PrivateAttr, SerializationInfo, TypeAdapter, field_validator, model_serializer, validator # pylint: disable=no-name-in-module
|
12
14
|
|
13
15
|
if tuple(sys.version_info[:2]) >= (3, 9):
|
14
16
|
# Python 3.9 and above
|
@@ -17,6 +19,8 @@ else:
|
|
17
19
|
# Python 3.8 and below
|
18
20
|
from typing_extensions import Annotated
|
19
21
|
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
|
20
24
|
|
21
25
|
class MapperType(str, Enum):
|
22
26
|
FIELD_MAPPING_SELECTOR = "field_mapping_selector"
|
@@ -860,3 +864,29 @@ StoredStreamConfiguration.model_rebuild()
|
|
860
864
|
InboundSyncStreamsConfiguration.model_rebuild()
|
861
865
|
StoredFieldMappings.model_rebuild()
|
862
866
|
OutboundSyncConfigurationParameters.model_rebuild()
|
867
|
+
|
868
|
+
|
869
|
+
def get_secrets(oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
|
870
|
+
) -> Dict[str, StoredConfigurationValue]:
|
871
|
+
connection_secrets = {}
|
872
|
+
import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
|
873
|
+
if oauth_secret_name is not None:
|
874
|
+
connection_secrets["access_token"] = StoredConfigurationValue(
|
875
|
+
value=_snowflake.get_oauth_access_token(oauth_secret_name)
|
876
|
+
)
|
877
|
+
if other_secrets_name is not None:
|
878
|
+
try:
|
879
|
+
secret_string_content = _snowflake.get_generic_secret_string(
|
880
|
+
other_secrets_name
|
881
|
+
)
|
882
|
+
if len(secret_string_content) > 2:
|
883
|
+
other_secrets = json.loads(secret_string_content)
|
884
|
+
connection_secrets = {
|
885
|
+
**connection_secrets,
|
886
|
+
**TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(other_secrets),
|
887
|
+
}
|
888
|
+
except Exception as exception:
|
889
|
+
logger.error(f"Error parsing secrets content for secret {other_secrets_name}: {str(exception)}")
|
890
|
+
raise ValueError(f"Error parsing secrets content: {str(exception)}") from exception
|
891
|
+
return connection_secrets
|
892
|
+
|
@@ -4,7 +4,9 @@ Omnata Plugin Runtime.
|
|
4
4
|
Includes data container classes and defines the contract for a plugin.
|
5
5
|
"""
|
6
6
|
from __future__ import annotations
|
7
|
+
from inspect import signature
|
7
8
|
import sys
|
9
|
+
from types import FunctionType
|
8
10
|
from typing import Union
|
9
11
|
if tuple(sys.version_info[:2]) >= (3, 9):
|
10
12
|
# Python 3.9 and above
|
@@ -22,6 +24,11 @@ import threading
|
|
22
24
|
import time
|
23
25
|
import hashlib
|
24
26
|
import requests
|
27
|
+
import pkgutil
|
28
|
+
import inspect
|
29
|
+
import importlib
|
30
|
+
import sys
|
31
|
+
import os
|
25
32
|
from abc import ABC, abstractmethod
|
26
33
|
from decimal import Decimal
|
27
34
|
from functools import partial, wraps, reduce
|
@@ -31,7 +38,7 @@ from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Type,
|
|
31
38
|
import jinja2
|
32
39
|
import pandas
|
33
40
|
from pydantic_core import to_jsonable_python
|
34
|
-
from pydantic import Field, TypeAdapter, root_validator, BaseModel
|
41
|
+
from pydantic import Field, TypeAdapter, ValidationError, create_model, root_validator, BaseModel
|
35
42
|
from dateutil.parser import parse
|
36
43
|
from jinja2 import Environment
|
37
44
|
from snowflake.connector.pandas_tools import write_pandas
|
@@ -65,6 +72,7 @@ from .configuration import (
|
|
65
72
|
StreamConfiguration,
|
66
73
|
SubscriptableBaseModel,
|
67
74
|
SyncConfigurationParameters,
|
75
|
+
get_secrets
|
68
76
|
)
|
69
77
|
from .forms import (
|
70
78
|
ConnectionMethod,
|
@@ -119,6 +127,8 @@ class PluginInfo(BaseModel):
|
|
119
127
|
Setting this to 'partner' means that the plugin was developed and distributed by a partner.
|
120
128
|
All other values only carry meaning for Omnata plugins, to indicate which iconography to apply within the application.
|
121
129
|
:param str package_source: Whether the plugin is packaged as a function or a stage
|
130
|
+
:param List[UDFDefinition] consumer_udfs: A list of UDFs that the plugin exposes to consumers
|
131
|
+
:param List[UDTFDefinition] consumer_udtfs: A list of UDTFs that the plugin exposes to consumers
|
122
132
|
"""
|
123
133
|
|
124
134
|
manifest: PluginManifest
|
@@ -131,6 +141,8 @@ class PluginInfo(BaseModel):
|
|
131
141
|
plugin_devkit_version: str = 'unknown'
|
132
142
|
tier: str
|
133
143
|
package_source: Literal["function", "stage"]
|
144
|
+
consumer_udfs: List[UDFDefinition] = Field(default_factory=list)
|
145
|
+
consumer_udtfs: List[UDTFDefinition] = Field(default_factory=list)
|
134
146
|
|
135
147
|
|
136
148
|
def jinja_filter(func):
|
@@ -2220,3 +2232,304 @@ def get_nested_value(nested_dict:Dict, keys:List[str]):
|
|
2220
2232
|
if isinstance(keys, str):
|
2221
2233
|
keys = [keys]
|
2222
2234
|
return reduce(lambda d, key: d.get(key) if isinstance(d, dict) else None, keys, nested_dict)
|
2235
|
+
|
2236
|
+
|
2237
|
+
class SnowflakeFunctionParameter(BaseModel):
|
2238
|
+
"""
|
2239
|
+
Represents a parameter for a Snowflake UDF or UDTF
|
2240
|
+
"""
|
2241
|
+
name: str
|
2242
|
+
description: str
|
2243
|
+
data_type: str
|
2244
|
+
default_value_clause: Optional[str] = None
|
2245
|
+
|
2246
|
+
def __str__(self):
|
2247
|
+
if self.default_value_clause:
|
2248
|
+
return f"{self.name} {self.data_type} default {self.default_value_clause}"
|
2249
|
+
return f"{self.name} {self.data_type}"
|
2250
|
+
|
2251
|
+
class SnowflakeUDTFResultColumn(BaseModel):
|
2252
|
+
"""
|
2253
|
+
Represents a result column for a Snowflake UDTF
|
2254
|
+
"""
|
2255
|
+
name: str
|
2256
|
+
data_type: str
|
2257
|
+
def __str__(self):
|
2258
|
+
return f"{self.name} {self.data_type}"
|
2259
|
+
|
2260
|
+
class UDTFDefinition(BaseModel):
|
2261
|
+
"""
|
2262
|
+
The information needed by the plugin uploader to put a Python UDTF definition into the setup script.
|
2263
|
+
Do not use this class directly in plugins, instead use the omnata_udtf decorator.
|
2264
|
+
"""
|
2265
|
+
name: str = Field(..., title="The name of the UDTF")
|
2266
|
+
language: Literal['python','java'] = Field(..., title="The language of the UDF")
|
2267
|
+
runtime_version: str = Field(..., title="The runtime version of the UDF (language dependent)")
|
2268
|
+
description: str = Field(..., title="A description of the UDTF")
|
2269
|
+
params: List[SnowflakeFunctionParameter] = Field(..., title="The parameters of the UDTF")
|
2270
|
+
result_columns: List[SnowflakeUDTFResultColumn] = Field(..., title="The result columns of the UDTF")
|
2271
|
+
handler: str = Field(..., title="The handler class/function for the UDTF")
|
2272
|
+
expose_to_consumer: bool = Field(..., title="Whether the UDTF should be exposed to consumers")
|
2273
|
+
imports: Optional[List[str]] = Field(None, title="A list of imports required by the UDF")
|
2274
|
+
packages: Optional[List[str]] = Field(None, title="A list of packages required by the UDTF")
|
2275
|
+
|
2276
|
+
def __str__(self):
|
2277
|
+
param_str = ', '.join([str(param) for param in self.params])
|
2278
|
+
table_result_columns = ', '.join([f"{col.name} {col.data_type}" for col in self.result_columns])
|
2279
|
+
packages_str = ', '.join([f"'{p}'" for p in self.packages])
|
2280
|
+
imports_str = ', '.join([f"'{i}'" for i in self.imports])
|
2281
|
+
return f"""CREATE OR REPLACE FUNCTION UDFS.{self.name}({param_str})
|
2282
|
+
RETURNS TABLE({table_result_columns})
|
2283
|
+
LANGUAGE {self.language.upper()}
|
2284
|
+
RUNTIME_VERSION={self.runtime_version}
|
2285
|
+
COMMENT = $${self.description}$$
|
2286
|
+
PACKAGES = ({packages_str})
|
2287
|
+
IMPORTS = ({imports_str})
|
2288
|
+
HANDLER='{self.handler}';
|
2289
|
+
"""
|
2290
|
+
|
2291
|
+
class UDFDefinition(BaseModel):
|
2292
|
+
"""
|
2293
|
+
The information needed by the plugin uploader to put a Python UDF definition into the setup script.
|
2294
|
+
Do not use this class directly in plugins, instead use the omnata_udf decorator.
|
2295
|
+
"""
|
2296
|
+
name: str = Field(..., title="The name of the UDF")
|
2297
|
+
language: Literal['python','java'] = Field(..., title="The language of the UDF")
|
2298
|
+
runtime_version: str = Field(..., title="The runtime version of the UDF (language dependent)")
|
2299
|
+
description: str = Field(..., title="A description of the UDF")
|
2300
|
+
params: List[SnowflakeFunctionParameter] = Field(..., title="The parameters of the UDF")
|
2301
|
+
result_data_type: str = Field(..., title="The data type returned by the UDF")
|
2302
|
+
handler: str = Field(..., title="The handler class/function for the UDF")
|
2303
|
+
expose_to_consumer: bool = Field(..., title="Whether the UDF should be exposed to consumers")
|
2304
|
+
imports: Optional[List[str]] = Field(None, title="A list of imports required by the UDF")
|
2305
|
+
packages: Optional[List[str]] = Field(None, title="A list of packages required by the UDF")
|
2306
|
+
|
2307
|
+
def __str__(self):
|
2308
|
+
param_str = ', '.join([str(param) for param in self.params])
|
2309
|
+
packages_str = ', '.join([f"'{p}'" for p in self.packages])
|
2310
|
+
imports_str = ', '.join([f"'{i}'" for i in self.imports])
|
2311
|
+
return f"""CREATE OR REPLACE FUNCTION UDFS.{self.name}({param_str})
|
2312
|
+
RETURNS {self.result_data_type}
|
2313
|
+
LANGUAGE {self.language.upper()}
|
2314
|
+
RUNTIME_VERSION={self.runtime_version}
|
2315
|
+
COMMENT = $${self.description}$$
|
2316
|
+
PACKAGES = ({packages_str})
|
2317
|
+
IMPORTS = ({imports_str})
|
2318
|
+
HANDLER='{self.handler}';
|
2319
|
+
"""
|
2320
|
+
|
2321
|
+
def omnata_udtf(
|
2322
|
+
name:str,
|
2323
|
+
description: str,
|
2324
|
+
params: List[SnowflakeFunctionParameter],
|
2325
|
+
result_columns: List[SnowflakeUDTFResultColumn],
|
2326
|
+
expose_to_consumer: bool):
|
2327
|
+
"""
|
2328
|
+
A decorator for a class which should create a UDTF in the UDFS schema of the native app
|
2329
|
+
"""
|
2330
|
+
def class_decorator(cls):
|
2331
|
+
# Get the original 'process' method from the class
|
2332
|
+
if not hasattr(cls, 'process'):
|
2333
|
+
raise ValueError("The class must have a 'process' method.")
|
2334
|
+
original_process = getattr(cls, 'process')
|
2335
|
+
sig = signature(original_process)
|
2336
|
+
function_params = sig.parameters
|
2337
|
+
if len(function_params) < 1:
|
2338
|
+
raise ValueError("The 'process' function must have at least one parameter.")
|
2339
|
+
|
2340
|
+
first_param_name = list(function_params.keys())[0]
|
2341
|
+
if first_param_name != 'self':
|
2342
|
+
raise ValueError(f"The first argument for the 'process' function should be 'self', instead it was '{first_param_name}'.")
|
2343
|
+
|
2344
|
+
cls._is_omnata_udtf = True
|
2345
|
+
cls._omnata_udtf_name = name
|
2346
|
+
cls._omnata_udtf_description = description
|
2347
|
+
cls._omnata_udtf_params = params
|
2348
|
+
cls._omnata_udtf_result_columns = result_columns
|
2349
|
+
cls._omnata_udtf_expose_to_consumer = expose_to_consumer
|
2350
|
+
|
2351
|
+
if not expose_to_consumer:
|
2352
|
+
# If not exposing to the consumer, there are no further requirements
|
2353
|
+
return cls
|
2354
|
+
|
2355
|
+
if len(function_params) < 2:
|
2356
|
+
raise ValueError("When exposing the udtf to consumers, the 'process' function must have the self parameter, plus at least the mandatory 'connection_parameters' parameter.")
|
2357
|
+
second_param_name = list(function_params.keys())[1]
|
2358
|
+
if second_param_name != 'connection_parameters':
|
2359
|
+
raise ValueError(f"The second argument should be 'connection_parameters', instead it was {second_param_name}.")
|
2360
|
+
if function_params[second_param_name].annotation != ConnectionConfigurationParameters:
|
2361
|
+
raise ValueError(f"The second argument must be a ConnectionConfigurationParameters, instead it was a {function_params[second_param_name].annotation}.")
|
2362
|
+
|
2363
|
+
if params[0].name.upper() != 'CONNECTION_PARAMETERS':
|
2364
|
+
params_new = [SnowflakeFunctionParameter(
|
2365
|
+
name='CONNECTION_PARAMETERS',
|
2366
|
+
data_type='OBJECT',
|
2367
|
+
description='The connection object, obtained from calling PLUGIN.PLUGIN_CONNECTION.')] + params
|
2368
|
+
cls._omnata_udtf_params = params_new
|
2369
|
+
if len(params_new) != len(function_params) -1:
|
2370
|
+
raise ValueError(f"You must document all the parameters of the 'process' function in the @omnata_udtf decorator in the same order ('connection_parameters' will be included automatically).")
|
2371
|
+
|
2372
|
+
@wraps(original_process)
|
2373
|
+
def wrapped_process(self, connection_parameter_arg, *args, **kwargs):
|
2374
|
+
if connection_parameter_arg is None:
|
2375
|
+
raise ValueError("Connection not found")
|
2376
|
+
|
2377
|
+
# convert the connection parameters dictionary to a ConnectionConfigurationParameters object which includes the real secrets
|
2378
|
+
if 'other_secrets_name' in connection_parameter_arg:
|
2379
|
+
# this is the new way, where the sync engine only passes the name of the secret
|
2380
|
+
oauth_secrets_name = None
|
2381
|
+
if 'oauth_secret_name' in connection_parameter_arg:
|
2382
|
+
oauth_secrets_name = connection_parameter_arg['oauth_secret_name']
|
2383
|
+
del connection_parameter_arg['oauth_secret_name']
|
2384
|
+
result = get_secrets(oauth_secrets_name,connection_parameter_arg['other_secrets_name'])
|
2385
|
+
connection_parameter_arg['connection_secrets'] = result
|
2386
|
+
del connection_parameter_arg['other_secrets_name']
|
2387
|
+
|
2388
|
+
parameters = ConnectionConfigurationParameters.model_validate(connection_parameter_arg)
|
2389
|
+
|
2390
|
+
# Pass the validated arguments to the function
|
2391
|
+
return original_process(self, parameters, *args, **kwargs)
|
2392
|
+
# Replace the original 'process' method with the wrapped version
|
2393
|
+
setattr(cls, 'process', wrapped_process)
|
2394
|
+
return cls
|
2395
|
+
|
2396
|
+
return class_decorator
|
2397
|
+
|
2398
|
+
def find_udtf_classes(path:str = '.') -> List[UDTFDefinition]:
|
2399
|
+
"""
|
2400
|
+
Finds all classes in the specified directory which have the 'omnata_udtf' decorator applied
|
2401
|
+
"""
|
2402
|
+
# Get the directory's absolute path
|
2403
|
+
current_dir = os.path.abspath(path)
|
2404
|
+
|
2405
|
+
# List to hold the classes that match the attribute
|
2406
|
+
matching_classes = []
|
2407
|
+
|
2408
|
+
# Iterate over all modules in the current directory
|
2409
|
+
for _, module_name, _ in pkgutil.iter_modules([current_dir]):
|
2410
|
+
# Import the module
|
2411
|
+
module = importlib.import_module(module_name)
|
2412
|
+
|
2413
|
+
# Iterate over all members of the module
|
2414
|
+
for name, obj in inspect.getmembers(module, inspect.isclass):
|
2415
|
+
# Check if the class has the specified attribute
|
2416
|
+
if hasattr(obj, '_is_omnata_udtf'):
|
2417
|
+
matching_classes.append(UDTFDefinition(
|
2418
|
+
name=obj._omnata_udtf_name,
|
2419
|
+
language='python',
|
2420
|
+
runtime_version='3.10',
|
2421
|
+
imports=['/app.zip'],
|
2422
|
+
description=obj._omnata_udtf_description,
|
2423
|
+
params=obj._omnata_udtf_params,
|
2424
|
+
result_columns=obj._omnata_udtf_result_columns,
|
2425
|
+
expose_to_consumer=obj._omnata_udtf_expose_to_consumer,
|
2426
|
+
handler=obj.__module__+'.'+obj.__name__
|
2427
|
+
))
|
2428
|
+
elif isinstance(obj, UDTFDefinition) and cast(UDTFDefinition,obj).language == 'java':
|
2429
|
+
matching_classes.append(obj)
|
2430
|
+
|
2431
|
+
return matching_classes
|
2432
|
+
|
2433
|
+
|
2434
|
+
def omnata_udf(
|
2435
|
+
name: str,
|
2436
|
+
description: str,
|
2437
|
+
params: List[SnowflakeFunctionParameter],
|
2438
|
+
result_data_type: str,
|
2439
|
+
expose_to_consumer: bool):
|
2440
|
+
"""
|
2441
|
+
A decorator for a function which will be created in the native application.
|
2442
|
+
"""
|
2443
|
+
def decorator(func):
|
2444
|
+
sig = signature(func)
|
2445
|
+
function_params = sig.parameters
|
2446
|
+
|
2447
|
+
if not expose_to_consumer:
|
2448
|
+
# If not exposing to the consumer, there are no further requirements
|
2449
|
+
func._is_omnata_udf = True
|
2450
|
+
func._omnata_udf_name = name
|
2451
|
+
func._omnata_udf_description = description
|
2452
|
+
func._omnata_udf_params = params
|
2453
|
+
func._omnata_udf_result_data_type = result_data_type
|
2454
|
+
func._omnata_udf_expose_to_consumer = expose_to_consumer
|
2455
|
+
return func
|
2456
|
+
|
2457
|
+
if len(function_params) == 0:
|
2458
|
+
raise ValueError("The function must have at least one parameter.")
|
2459
|
+
# Ensure the first argument is mandatory and positional
|
2460
|
+
first_param_name = list(function_params.keys())[0]
|
2461
|
+
if first_param_name != 'connection_parameters':
|
2462
|
+
raise ValueError(f"The first argument should be 'connection_parameters', instead it was '{first_param_name}'.")
|
2463
|
+
if function_params[first_param_name].annotation != ConnectionConfigurationParameters:
|
2464
|
+
raise ValueError(f"The first argument must be a ConnectionConfigurationParameters, instead it was a {function_params[first_param_name].annotation}.")
|
2465
|
+
if params[0].name.upper() != 'CONNECTION_PARAMETERS':
|
2466
|
+
params_new = [SnowflakeFunctionParameter(
|
2467
|
+
name='CONNECTION_PARAMETERS',
|
2468
|
+
data_type='OBJECT',
|
2469
|
+
description='The connection object, obtained from calling PLUGIN.PLUGIN_CONNECTION.')] + params
|
2470
|
+
func._omnata_udf_params = params_new
|
2471
|
+
if len(params_new) != len(function_params):
|
2472
|
+
raise ValueError(f"You must document all the parameters of the function in the @omnata_udf decorator in the same order ('connection_parameters' will be included automatically).")
|
2473
|
+
|
2474
|
+
@wraps(func)
|
2475
|
+
def wrapper(connection_parameter_arg, *args, **kwargs):
|
2476
|
+
# convert the connection parameters dictionary to a ConnectionConfigurationParameters object which includes the real secrets
|
2477
|
+
if 'other_secrets_name' in connection_parameter_arg:
|
2478
|
+
# this is the new way, where the sync engine only passes the name of the secret
|
2479
|
+
oauth_secrets_name = None
|
2480
|
+
if 'oauth_secret_name' in connection_parameter_arg:
|
2481
|
+
oauth_secrets_name = connection_parameter_arg['oauth_secret_name']
|
2482
|
+
del connection_parameter_arg['oauth_secret_name']
|
2483
|
+
result = get_secrets(oauth_secrets_name,connection_parameter_arg['other_secrets_name'])
|
2484
|
+
connection_parameter_arg['connection_secrets'] = result
|
2485
|
+
del connection_parameter_arg['other_secrets_name']
|
2486
|
+
|
2487
|
+
parameters = ConnectionConfigurationParameters.model_validate(connection_parameter_arg)
|
2488
|
+
|
2489
|
+
# Pass the validated arguments to the function
|
2490
|
+
return func(parameters, *args, **kwargs)
|
2491
|
+
|
2492
|
+
wrapper._is_omnata_udf = True
|
2493
|
+
wrapper._omnata_udf_name = name
|
2494
|
+
wrapper._omnata_udf_description = description
|
2495
|
+
wrapper._omnata_udf_result_data_type = result_data_type
|
2496
|
+
wrapper._omnata_udf_expose_to_consumer = expose_to_consumer
|
2497
|
+
return wrapper
|
2498
|
+
|
2499
|
+
return decorator
|
2500
|
+
|
2501
|
+
def find_udf_functions(path:str = '.') -> List[UDFDefinition]:
|
2502
|
+
"""
|
2503
|
+
Finds all functions in the specified directory which have the 'omnata_udf' decorator applied
|
2504
|
+
"""
|
2505
|
+
# Get the current directory's absolute path
|
2506
|
+
current_dir = os.path.abspath(path)
|
2507
|
+
|
2508
|
+
# List to hold the classes that match the attribute
|
2509
|
+
matching_classes = []
|
2510
|
+
|
2511
|
+
# Iterate over all modules in the current directory
|
2512
|
+
for _, module_name, _ in pkgutil.iter_modules([current_dir]):
|
2513
|
+
# Import the module
|
2514
|
+
module = importlib.import_module(module_name)
|
2515
|
+
|
2516
|
+
# Iterate over all members of the module
|
2517
|
+
for name, obj in inspect.getmembers(module, inspect.isfunction):
|
2518
|
+
# Check if the class has the specified attribute
|
2519
|
+
if hasattr(obj, '_is_omnata_udf'):
|
2520
|
+
matching_classes.append(UDFDefinition(
|
2521
|
+
name=obj._omnata_udf_name,
|
2522
|
+
language='python',
|
2523
|
+
runtime_version='3.10',
|
2524
|
+
imports=['/app.zip'],
|
2525
|
+
description=obj._omnata_udf_description,
|
2526
|
+
params=obj._omnata_udf_params,
|
2527
|
+
result_data_type=obj._omnata_udf_result_data_type,
|
2528
|
+
expose_to_consumer=obj._omnata_udf_expose_to_consumer,
|
2529
|
+
packages=[],
|
2530
|
+
handler=obj.__module__+'.'+obj.__name__
|
2531
|
+
))
|
2532
|
+
elif isinstance(obj, UDFDefinition) and cast(UDFDefinition,obj).language == 'java':
|
2533
|
+
matching_classes.append(obj)
|
2534
|
+
|
2535
|
+
return matching_classes
|
@@ -20,6 +20,7 @@ from .configuration import (
|
|
20
20
|
OutboundSyncStrategy,
|
21
21
|
StoredConfigurationValue,
|
22
22
|
StoredMappingValue,
|
23
|
+
get_secrets
|
23
24
|
)
|
24
25
|
from .forms import ConnectionMethod, FormInputField, FormOption
|
25
26
|
from .logging import OmnataPluginLogHandler
|
@@ -71,7 +72,7 @@ class PluginEntrypoint:
|
|
71
72
|
def sync(self, sync_request: Dict):
|
72
73
|
logger.info("Entered sync method")
|
73
74
|
request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
|
74
|
-
connection_secrets =
|
75
|
+
connection_secrets = get_secrets(
|
75
76
|
request.oauth_secret_name, request.other_secrets_name
|
76
77
|
)
|
77
78
|
omnata_log_handler = OmnataPluginLogHandler(
|
@@ -270,7 +271,7 @@ class PluginEntrypoint:
|
|
270
271
|
sync_strategy = normalise_nulls(sync_strategy)
|
271
272
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
272
273
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
273
|
-
connection_secrets =
|
274
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
274
275
|
connection_parameters = TypeAdapter(
|
275
276
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
276
277
|
sync_parameters = TypeAdapter(
|
@@ -324,7 +325,7 @@ class PluginEntrypoint:
|
|
324
325
|
logger.info("Entered list_streams method")
|
325
326
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
326
327
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
327
|
-
connection_secrets =
|
328
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
328
329
|
connection_parameters = TypeAdapter(
|
329
330
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
330
331
|
sync_parameters = TypeAdapter(
|
@@ -411,31 +412,6 @@ class PluginEntrypoint:
|
|
411
412
|
raise e
|
412
413
|
return [e.model_dump() for e in events]
|
413
414
|
|
414
|
-
def get_secrets(
|
415
|
-
self, oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
|
416
|
-
) -> Dict[str, StoredConfigurationValue]:
|
417
|
-
connection_secrets = {}
|
418
|
-
import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
|
419
|
-
if oauth_secret_name is not None:
|
420
|
-
connection_secrets["access_token"] = StoredConfigurationValue(
|
421
|
-
value=_snowflake.get_oauth_access_token(oauth_secret_name)
|
422
|
-
)
|
423
|
-
if other_secrets_name is not None:
|
424
|
-
try:
|
425
|
-
secret_string_content = _snowflake.get_generic_secret_string(
|
426
|
-
other_secrets_name
|
427
|
-
)
|
428
|
-
if len(secret_string_content) > 2:
|
429
|
-
other_secrets = json.loads(secret_string_content)
|
430
|
-
connection_secrets = {
|
431
|
-
**connection_secrets,
|
432
|
-
**TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(other_secrets),
|
433
|
-
}
|
434
|
-
except Exception as exception:
|
435
|
-
logger.error(f"Error parsing secrets content for secret {other_secrets_name}: {str(exception)}")
|
436
|
-
raise ValueError(f"Error parsing secrets content: {str(exception)}") from exception
|
437
|
-
return connection_secrets
|
438
|
-
|
439
415
|
def ngrok_post_tunnel_fields(
|
440
416
|
self,
|
441
417
|
connection_method: str,
|
@@ -447,7 +423,7 @@ class PluginEntrypoint:
|
|
447
423
|
logger.info("Entered ngrok_post_tunnel_fields method")
|
448
424
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
449
425
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
450
|
-
connection_secrets =
|
426
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
451
427
|
connection_parameters = TypeAdapter(
|
452
428
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
453
429
|
parameters = ConnectionConfigurationParameters(
|
@@ -493,7 +469,7 @@ class PluginEntrypoint:
|
|
493
469
|
):
|
494
470
|
logger.info("Entered connect method")
|
495
471
|
logger.info(f"Connection parameters: {connection_parameters}")
|
496
|
-
connection_secrets =
|
472
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
497
473
|
|
498
474
|
from omnata_plugin_runtime.omnata_plugin import (
|
499
475
|
ConnectionConfigurationParameters,
|
@@ -540,7 +516,7 @@ class PluginEntrypoint:
|
|
540
516
|
oauth_secret_name: Optional[str],
|
541
517
|
other_secrets_name: Optional[str]):
|
542
518
|
logger.info("Entered api_limits method")
|
543
|
-
connection_secrets =
|
519
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
544
520
|
from omnata_plugin_runtime.omnata_plugin import (
|
545
521
|
ConnectionConfigurationParameters,
|
546
522
|
)
|
File without changes
|
File without changes
|
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/api.py
RENAMED
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/forms.py
RENAMED
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/logging.py
RENAMED
File without changes
|
File without changes
|