omnata-plugin-runtime 0.5.5a127__tar.gz → 0.5.6__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/PKG-INFO +2 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/pyproject.toml +2 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/configuration.py +31 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/omnata_plugin.py +314 -1
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/plugin_entrypoints.py +7 -31
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/LICENSE +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/README.md +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/__init__.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/api.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/forms.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/logging.py +0 -0
- {omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/rate_limiting.py +0 -0
@@ -1,6 +1,6 @@
|
|
1
1
|
Metadata-Version: 2.1
|
2
2
|
Name: omnata-plugin-runtime
|
3
|
-
Version: 0.5.
|
3
|
+
Version: 0.5.6
|
4
4
|
Summary: Classes and common runtime components for building and running Omnata Plugins
|
5
5
|
Author: James Weakley
|
6
6
|
Author-email: james.weakley@omnata.com
|
@@ -22,6 +22,7 @@ Requires-Dist: numpy (<=1.26.4)
|
|
22
22
|
Requires-Dist: packaging (<=24.1)
|
23
23
|
Requires-Dist: pandas (<=2.2.2)
|
24
24
|
Requires-Dist: platformdirs (<=3.10.0)
|
25
|
+
Requires-Dist: pyarrow (<=16.1.0)
|
25
26
|
Requires-Dist: pycparser (<=2.21)
|
26
27
|
Requires-Dist: pydantic (>=2,<=2.5.3)
|
27
28
|
Requires-Dist: pydantic-core (<=2.14.6)
|
@@ -1,6 +1,6 @@
|
|
1
1
|
[tool.poetry]
|
2
2
|
name = "omnata-plugin-runtime"
|
3
|
-
version = "0.5.
|
3
|
+
version = "0.5.6"
|
4
4
|
description = "Classes and common runtime components for building and running Omnata Plugins"
|
5
5
|
authors = ["James Weakley <james.weakley@omnata.com>"]
|
6
6
|
readme = "README.md"
|
@@ -37,6 +37,7 @@ urllib3 = "<=2.2.2" # latest version available on Snowflake Anaconda
|
|
37
37
|
wheel = "<=0.43.0" # latest version available on Snowflake Anaconda
|
38
38
|
pyyaml = "<=6.0.1" # latest version available on Snowflake Anaconda
|
39
39
|
cffi = "<=1.16.0" # latest version available on Snowflake Anaconda
|
40
|
+
pyarrow = "<=16.1.0" # latest version available on Snowflake Anaconda
|
40
41
|
|
41
42
|
[tool.poetry.dev-dependencies]
|
42
43
|
pytest = "^6.2.4"
|
@@ -3,12 +3,14 @@ Omnata Plugin Runtime configuration objects.
|
|
3
3
|
Includes data container classes related to plugin configuration.
|
4
4
|
"""
|
5
5
|
from __future__ import annotations
|
6
|
+
import json
|
6
7
|
import sys
|
8
|
+
import logging
|
7
9
|
from typing import Any, List, Dict, Literal, Union, Optional
|
8
10
|
from enum import Enum
|
9
11
|
|
10
12
|
from abc import ABC
|
11
|
-
from pydantic import BaseModel, Field, PrivateAttr, SerializationInfo, field_validator, model_serializer, validator # pylint: disable=no-name-in-module
|
13
|
+
from pydantic import BaseModel, Field, PrivateAttr, SerializationInfo, TypeAdapter, field_validator, model_serializer, validator # pylint: disable=no-name-in-module
|
12
14
|
|
13
15
|
if tuple(sys.version_info[:2]) >= (3, 9):
|
14
16
|
# Python 3.9 and above
|
@@ -17,6 +19,8 @@ else:
|
|
17
19
|
# Python 3.8 and below
|
18
20
|
from typing_extensions import Annotated
|
19
21
|
|
22
|
+
logger = logging.getLogger(__name__)
|
23
|
+
|
20
24
|
|
21
25
|
class MapperType(str, Enum):
|
22
26
|
FIELD_MAPPING_SELECTOR = "field_mapping_selector"
|
@@ -860,3 +864,29 @@ StoredStreamConfiguration.model_rebuild()
|
|
860
864
|
InboundSyncStreamsConfiguration.model_rebuild()
|
861
865
|
StoredFieldMappings.model_rebuild()
|
862
866
|
OutboundSyncConfigurationParameters.model_rebuild()
|
867
|
+
|
868
|
+
|
869
|
+
def get_secrets(oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
|
870
|
+
) -> Dict[str, StoredConfigurationValue]:
|
871
|
+
connection_secrets = {}
|
872
|
+
import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
|
873
|
+
if oauth_secret_name is not None:
|
874
|
+
connection_secrets["access_token"] = StoredConfigurationValue(
|
875
|
+
value=_snowflake.get_oauth_access_token(oauth_secret_name)
|
876
|
+
)
|
877
|
+
if other_secrets_name is not None:
|
878
|
+
try:
|
879
|
+
secret_string_content = _snowflake.get_generic_secret_string(
|
880
|
+
other_secrets_name
|
881
|
+
)
|
882
|
+
if len(secret_string_content) > 2:
|
883
|
+
other_secrets = json.loads(secret_string_content)
|
884
|
+
connection_secrets = {
|
885
|
+
**connection_secrets,
|
886
|
+
**TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(other_secrets),
|
887
|
+
}
|
888
|
+
except Exception as exception:
|
889
|
+
logger.error(f"Error parsing secrets content for secret {other_secrets_name}: {str(exception)}")
|
890
|
+
raise ValueError(f"Error parsing secrets content: {str(exception)}") from exception
|
891
|
+
return connection_secrets
|
892
|
+
|
@@ -4,7 +4,9 @@ Omnata Plugin Runtime.
|
|
4
4
|
Includes data container classes and defines the contract for a plugin.
|
5
5
|
"""
|
6
6
|
from __future__ import annotations
|
7
|
+
from inspect import signature
|
7
8
|
import sys
|
9
|
+
from types import FunctionType
|
8
10
|
from typing import Union
|
9
11
|
if tuple(sys.version_info[:2]) >= (3, 9):
|
10
12
|
# Python 3.9 and above
|
@@ -22,6 +24,11 @@ import threading
|
|
22
24
|
import time
|
23
25
|
import hashlib
|
24
26
|
import requests
|
27
|
+
import pkgutil
|
28
|
+
import inspect
|
29
|
+
import importlib
|
30
|
+
import sys
|
31
|
+
import os
|
25
32
|
from abc import ABC, abstractmethod
|
26
33
|
from decimal import Decimal
|
27
34
|
from functools import partial, wraps, reduce
|
@@ -31,7 +38,7 @@ from typing import Any, Callable, Dict, Iterable, List, Literal, Optional, Type,
|
|
31
38
|
import jinja2
|
32
39
|
import pandas
|
33
40
|
from pydantic_core import to_jsonable_python
|
34
|
-
from pydantic import Field, TypeAdapter, root_validator, BaseModel
|
41
|
+
from pydantic import Field, TypeAdapter, ValidationError, create_model, root_validator, BaseModel
|
35
42
|
from dateutil.parser import parse
|
36
43
|
from jinja2 import Environment
|
37
44
|
from snowflake.connector.pandas_tools import write_pandas
|
@@ -65,6 +72,7 @@ from .configuration import (
|
|
65
72
|
StreamConfiguration,
|
66
73
|
SubscriptableBaseModel,
|
67
74
|
SyncConfigurationParameters,
|
75
|
+
get_secrets
|
68
76
|
)
|
69
77
|
from .forms import (
|
70
78
|
ConnectionMethod,
|
@@ -119,6 +127,8 @@ class PluginInfo(BaseModel):
|
|
119
127
|
Setting this to 'partner' means that the plugin was developed and distributed by a partner.
|
120
128
|
All other values only carry meaning for Omnata plugins, to indicate which iconography to apply within the application.
|
121
129
|
:param str package_source: Whether the plugin is packaged as a function or a stage
|
130
|
+
:param List[UDFDefinition] consumer_udfs: A list of UDFs that the plugin exposes to consumers
|
131
|
+
:param List[UDTFDefinition] consumer_udtfs: A list of UDTFs that the plugin exposes to consumers
|
122
132
|
"""
|
123
133
|
|
124
134
|
manifest: PluginManifest
|
@@ -131,6 +141,8 @@ class PluginInfo(BaseModel):
|
|
131
141
|
plugin_devkit_version: str = 'unknown'
|
132
142
|
tier: str
|
133
143
|
package_source: Literal["function", "stage"]
|
144
|
+
consumer_udfs: List[UDFDefinition] = Field(default_factory=list)
|
145
|
+
consumer_udtfs: List[UDTFDefinition] = Field(default_factory=list)
|
134
146
|
|
135
147
|
|
136
148
|
def jinja_filter(func):
|
@@ -2220,3 +2232,304 @@ def get_nested_value(nested_dict:Dict, keys:List[str]):
|
|
2220
2232
|
if isinstance(keys, str):
|
2221
2233
|
keys = [keys]
|
2222
2234
|
return reduce(lambda d, key: d.get(key) if isinstance(d, dict) else None, keys, nested_dict)
|
2235
|
+
|
2236
|
+
|
2237
|
+
class SnowflakeFunctionParameter(BaseModel):
|
2238
|
+
"""
|
2239
|
+
Represents a parameter for a Snowflake UDF or UDTF
|
2240
|
+
"""
|
2241
|
+
name: str
|
2242
|
+
description: str
|
2243
|
+
data_type: str
|
2244
|
+
default_value_clause: Optional[str] = None
|
2245
|
+
|
2246
|
+
def __str__(self):
|
2247
|
+
if self.default_value_clause:
|
2248
|
+
return f"{self.name} {self.data_type} default {self.default_value_clause}"
|
2249
|
+
return f"{self.name} {self.data_type}"
|
2250
|
+
|
2251
|
+
class SnowflakeUDTFResultColumn(BaseModel):
|
2252
|
+
"""
|
2253
|
+
Represents a result column for a Snowflake UDTF
|
2254
|
+
"""
|
2255
|
+
name: str
|
2256
|
+
data_type: str
|
2257
|
+
def __str__(self):
|
2258
|
+
return f"{self.name} {self.data_type}"
|
2259
|
+
|
2260
|
+
class UDTFDefinition(BaseModel):
|
2261
|
+
"""
|
2262
|
+
The information needed by the plugin uploader to put a Python UDTF definition into the setup script.
|
2263
|
+
Do not use this class directly in plugins, instead use the omnata_udtf decorator.
|
2264
|
+
"""
|
2265
|
+
name: str = Field(..., title="The name of the UDTF")
|
2266
|
+
language: Literal['python','java'] = Field(..., title="The language of the UDF")
|
2267
|
+
runtime_version: str = Field(..., title="The runtime version of the UDF (language dependent)")
|
2268
|
+
description: str = Field(..., title="A description of the UDTF")
|
2269
|
+
params: List[SnowflakeFunctionParameter] = Field(..., title="The parameters of the UDTF")
|
2270
|
+
result_columns: List[SnowflakeUDTFResultColumn] = Field(..., title="The result columns of the UDTF")
|
2271
|
+
handler: str = Field(..., title="The handler class/function for the UDTF")
|
2272
|
+
expose_to_consumer: bool = Field(..., title="Whether the UDTF should be exposed to consumers")
|
2273
|
+
imports: Optional[List[str]] = Field(None, title="A list of imports required by the UDF")
|
2274
|
+
packages: Optional[List[str]] = Field(None, title="A list of packages required by the UDTF")
|
2275
|
+
|
2276
|
+
def __str__(self):
|
2277
|
+
param_str = ', '.join([str(param) for param in self.params])
|
2278
|
+
table_result_columns = ', '.join([f"{col.name} {col.data_type}" for col in self.result_columns])
|
2279
|
+
packages_str = ', '.join([f"'{p}'" for p in self.packages])
|
2280
|
+
imports_str = ', '.join([f"'{i}'" for i in self.imports])
|
2281
|
+
return f"""CREATE OR REPLACE FUNCTION UDFS.{self.name}({param_str})
|
2282
|
+
RETURNS TABLE({table_result_columns})
|
2283
|
+
LANGUAGE {self.language.upper()}
|
2284
|
+
RUNTIME_VERSION={self.runtime_version}
|
2285
|
+
COMMENT = $${self.description}$$
|
2286
|
+
PACKAGES = ({packages_str})
|
2287
|
+
IMPORTS = ({imports_str})
|
2288
|
+
HANDLER='{self.handler}';
|
2289
|
+
"""
|
2290
|
+
|
2291
|
+
class UDFDefinition(BaseModel):
|
2292
|
+
"""
|
2293
|
+
The information needed by the plugin uploader to put a Python UDF definition into the setup script.
|
2294
|
+
Do not use this class directly in plugins, instead use the omnata_udf decorator.
|
2295
|
+
"""
|
2296
|
+
name: str = Field(..., title="The name of the UDF")
|
2297
|
+
language: Literal['python','java'] = Field(..., title="The language of the UDF")
|
2298
|
+
runtime_version: str = Field(..., title="The runtime version of the UDF (language dependent)")
|
2299
|
+
description: str = Field(..., title="A description of the UDF")
|
2300
|
+
params: List[SnowflakeFunctionParameter] = Field(..., title="The parameters of the UDF")
|
2301
|
+
result_data_type: str = Field(..., title="The data type returned by the UDF")
|
2302
|
+
handler: str = Field(..., title="The handler class/function for the UDF")
|
2303
|
+
expose_to_consumer: bool = Field(..., title="Whether the UDF should be exposed to consumers")
|
2304
|
+
imports: Optional[List[str]] = Field(None, title="A list of imports required by the UDF")
|
2305
|
+
packages: Optional[List[str]] = Field(None, title="A list of packages required by the UDF")
|
2306
|
+
|
2307
|
+
def __str__(self):
|
2308
|
+
param_str = ', '.join([str(param) for param in self.params])
|
2309
|
+
packages_str = ', '.join([f"'{p}'" for p in self.packages])
|
2310
|
+
imports_str = ', '.join([f"'{i}'" for i in self.imports])
|
2311
|
+
return f"""CREATE OR REPLACE FUNCTION UDFS.{self.name}({param_str})
|
2312
|
+
RETURNS {self.result_data_type}
|
2313
|
+
LANGUAGE {self.language.upper()}
|
2314
|
+
RUNTIME_VERSION={self.runtime_version}
|
2315
|
+
COMMENT = $${self.description}$$
|
2316
|
+
PACKAGES = ({packages_str})
|
2317
|
+
IMPORTS = ({imports_str})
|
2318
|
+
HANDLER='{self.handler}';
|
2319
|
+
"""
|
2320
|
+
|
2321
|
+
def omnata_udtf(
|
2322
|
+
name:str,
|
2323
|
+
description: str,
|
2324
|
+
params: List[SnowflakeFunctionParameter],
|
2325
|
+
result_columns: List[SnowflakeUDTFResultColumn],
|
2326
|
+
expose_to_consumer: bool):
|
2327
|
+
"""
|
2328
|
+
A decorator for a class which should create a UDTF in the UDFS schema of the native app
|
2329
|
+
"""
|
2330
|
+
def class_decorator(cls):
|
2331
|
+
# Get the original 'process' method from the class
|
2332
|
+
if not hasattr(cls, 'process'):
|
2333
|
+
raise ValueError("The class must have a 'process' method.")
|
2334
|
+
original_process = getattr(cls, 'process')
|
2335
|
+
sig = signature(original_process)
|
2336
|
+
function_params = sig.parameters
|
2337
|
+
if len(function_params) < 1:
|
2338
|
+
raise ValueError("The 'process' function must have at least one parameter.")
|
2339
|
+
|
2340
|
+
first_param_name = list(function_params.keys())[0]
|
2341
|
+
if first_param_name != 'self':
|
2342
|
+
raise ValueError(f"The first argument for the 'process' function should be 'self', instead it was '{first_param_name}'.")
|
2343
|
+
|
2344
|
+
cls._is_omnata_udtf = True
|
2345
|
+
cls._omnata_udtf_name = name
|
2346
|
+
cls._omnata_udtf_description = description
|
2347
|
+
cls._omnata_udtf_params = params
|
2348
|
+
cls._omnata_udtf_result_columns = result_columns
|
2349
|
+
cls._omnata_udtf_expose_to_consumer = expose_to_consumer
|
2350
|
+
|
2351
|
+
if not expose_to_consumer:
|
2352
|
+
# If not exposing to the consumer, there are no further requirements
|
2353
|
+
return cls
|
2354
|
+
|
2355
|
+
if len(function_params) < 2:
|
2356
|
+
raise ValueError("When exposing the udtf to consumers, the 'process' function must have the self parameter, plus at least the mandatory 'connection_parameters' parameter.")
|
2357
|
+
second_param_name = list(function_params.keys())[1]
|
2358
|
+
if second_param_name != 'connection_parameters':
|
2359
|
+
raise ValueError(f"The second argument should be 'connection_parameters', instead it was {second_param_name}.")
|
2360
|
+
if function_params[second_param_name].annotation != ConnectionConfigurationParameters:
|
2361
|
+
raise ValueError(f"The second argument must be a ConnectionConfigurationParameters, instead it was a {function_params[second_param_name].annotation}.")
|
2362
|
+
|
2363
|
+
if params[0].name.upper() != 'CONNECTION_PARAMETERS':
|
2364
|
+
params_new = [SnowflakeFunctionParameter(
|
2365
|
+
name='CONNECTION_PARAMETERS',
|
2366
|
+
data_type='OBJECT',
|
2367
|
+
description='The connection object, obtained from calling PLUGIN.PLUGIN_CONNECTION.')] + params
|
2368
|
+
cls._omnata_udtf_params = params_new
|
2369
|
+
if len(params_new) != len(function_params) -1:
|
2370
|
+
raise ValueError(f"You must document all the parameters of the 'process' function in the @omnata_udtf decorator in the same order ('connection_parameters' will be included automatically).")
|
2371
|
+
|
2372
|
+
@wraps(original_process)
|
2373
|
+
def wrapped_process(self, connection_parameter_arg, *args, **kwargs):
|
2374
|
+
if connection_parameter_arg is None:
|
2375
|
+
raise ValueError("Connection not found")
|
2376
|
+
|
2377
|
+
# convert the connection parameters dictionary to a ConnectionConfigurationParameters object which includes the real secrets
|
2378
|
+
if 'other_secrets_name' in connection_parameter_arg:
|
2379
|
+
# this is the new way, where the sync engine only passes the name of the secret
|
2380
|
+
oauth_secrets_name = None
|
2381
|
+
if 'oauth_secret_name' in connection_parameter_arg:
|
2382
|
+
oauth_secrets_name = connection_parameter_arg['oauth_secret_name']
|
2383
|
+
del connection_parameter_arg['oauth_secret_name']
|
2384
|
+
result = get_secrets(oauth_secrets_name,connection_parameter_arg['other_secrets_name'])
|
2385
|
+
connection_parameter_arg['connection_secrets'] = result
|
2386
|
+
del connection_parameter_arg['other_secrets_name']
|
2387
|
+
|
2388
|
+
parameters = ConnectionConfigurationParameters.model_validate(connection_parameter_arg)
|
2389
|
+
|
2390
|
+
# Pass the validated arguments to the function
|
2391
|
+
return original_process(self, parameters, *args, **kwargs)
|
2392
|
+
# Replace the original 'process' method with the wrapped version
|
2393
|
+
setattr(cls, 'process', wrapped_process)
|
2394
|
+
return cls
|
2395
|
+
|
2396
|
+
return class_decorator
|
2397
|
+
|
2398
|
+
def find_udtf_classes(path:str = '.') -> List[UDTFDefinition]:
|
2399
|
+
"""
|
2400
|
+
Finds all classes in the specified directory which have the 'omnata_udtf' decorator applied
|
2401
|
+
"""
|
2402
|
+
# Get the directory's absolute path
|
2403
|
+
current_dir = os.path.abspath(path)
|
2404
|
+
|
2405
|
+
# List to hold the classes that match the attribute
|
2406
|
+
matching_classes = []
|
2407
|
+
|
2408
|
+
# Iterate over all modules in the current directory
|
2409
|
+
for _, module_name, _ in pkgutil.iter_modules([current_dir]):
|
2410
|
+
# Import the module
|
2411
|
+
module = importlib.import_module(module_name)
|
2412
|
+
|
2413
|
+
# Iterate over all members of the module
|
2414
|
+
for name, obj in inspect.getmembers(module, inspect.isclass):
|
2415
|
+
# Check if the class has the specified attribute
|
2416
|
+
if hasattr(obj, '_is_omnata_udtf'):
|
2417
|
+
matching_classes.append(UDTFDefinition(
|
2418
|
+
name=obj._omnata_udtf_name,
|
2419
|
+
language='python',
|
2420
|
+
runtime_version='3.10',
|
2421
|
+
imports=['/app.zip'],
|
2422
|
+
description=obj._omnata_udtf_description,
|
2423
|
+
params=obj._omnata_udtf_params,
|
2424
|
+
result_columns=obj._omnata_udtf_result_columns,
|
2425
|
+
expose_to_consumer=obj._omnata_udtf_expose_to_consumer,
|
2426
|
+
handler=obj.__module__+'.'+obj.__name__
|
2427
|
+
))
|
2428
|
+
elif isinstance(obj, UDTFDefinition) and cast(UDTFDefinition,obj).language == 'java':
|
2429
|
+
matching_classes.append(obj)
|
2430
|
+
|
2431
|
+
return matching_classes
|
2432
|
+
|
2433
|
+
|
2434
|
+
def omnata_udf(
|
2435
|
+
name: str,
|
2436
|
+
description: str,
|
2437
|
+
params: List[SnowflakeFunctionParameter],
|
2438
|
+
result_data_type: str,
|
2439
|
+
expose_to_consumer: bool):
|
2440
|
+
"""
|
2441
|
+
A decorator for a function which will be created in the native application.
|
2442
|
+
"""
|
2443
|
+
def decorator(func):
|
2444
|
+
sig = signature(func)
|
2445
|
+
function_params = sig.parameters
|
2446
|
+
|
2447
|
+
if not expose_to_consumer:
|
2448
|
+
# If not exposing to the consumer, there are no further requirements
|
2449
|
+
func._is_omnata_udf = True
|
2450
|
+
func._omnata_udf_name = name
|
2451
|
+
func._omnata_udf_description = description
|
2452
|
+
func._omnata_udf_params = params
|
2453
|
+
func._omnata_udf_result_data_type = result_data_type
|
2454
|
+
func._omnata_udf_expose_to_consumer = expose_to_consumer
|
2455
|
+
return func
|
2456
|
+
|
2457
|
+
if len(function_params) == 0:
|
2458
|
+
raise ValueError("The function must have at least one parameter.")
|
2459
|
+
# Ensure the first argument is mandatory and positional
|
2460
|
+
first_param_name = list(function_params.keys())[0]
|
2461
|
+
if first_param_name != 'connection_parameters':
|
2462
|
+
raise ValueError(f"The first argument should be 'connection_parameters', instead it was '{first_param_name}'.")
|
2463
|
+
if function_params[first_param_name].annotation != ConnectionConfigurationParameters:
|
2464
|
+
raise ValueError(f"The first argument must be a ConnectionConfigurationParameters, instead it was a {function_params[first_param_name].annotation}.")
|
2465
|
+
if params[0].name.upper() != 'CONNECTION_PARAMETERS':
|
2466
|
+
params_new = [SnowflakeFunctionParameter(
|
2467
|
+
name='CONNECTION_PARAMETERS',
|
2468
|
+
data_type='OBJECT',
|
2469
|
+
description='The connection object, obtained from calling PLUGIN.PLUGIN_CONNECTION.')] + params
|
2470
|
+
func._omnata_udf_params = params_new
|
2471
|
+
if len(params_new) != len(function_params):
|
2472
|
+
raise ValueError(f"You must document all the parameters of the function in the @omnata_udf decorator in the same order ('connection_parameters' will be included automatically).")
|
2473
|
+
|
2474
|
+
@wraps(func)
|
2475
|
+
def wrapper(connection_parameter_arg, *args, **kwargs):
|
2476
|
+
# convert the connection parameters dictionary to a ConnectionConfigurationParameters object which includes the real secrets
|
2477
|
+
if 'other_secrets_name' in connection_parameter_arg:
|
2478
|
+
# this is the new way, where the sync engine only passes the name of the secret
|
2479
|
+
oauth_secrets_name = None
|
2480
|
+
if 'oauth_secret_name' in connection_parameter_arg:
|
2481
|
+
oauth_secrets_name = connection_parameter_arg['oauth_secret_name']
|
2482
|
+
del connection_parameter_arg['oauth_secret_name']
|
2483
|
+
result = get_secrets(oauth_secrets_name,connection_parameter_arg['other_secrets_name'])
|
2484
|
+
connection_parameter_arg['connection_secrets'] = result
|
2485
|
+
del connection_parameter_arg['other_secrets_name']
|
2486
|
+
|
2487
|
+
parameters = ConnectionConfigurationParameters.model_validate(connection_parameter_arg)
|
2488
|
+
|
2489
|
+
# Pass the validated arguments to the function
|
2490
|
+
return func(parameters, *args, **kwargs)
|
2491
|
+
|
2492
|
+
wrapper._is_omnata_udf = True
|
2493
|
+
wrapper._omnata_udf_name = name
|
2494
|
+
wrapper._omnata_udf_description = description
|
2495
|
+
wrapper._omnata_udf_result_data_type = result_data_type
|
2496
|
+
wrapper._omnata_udf_expose_to_consumer = expose_to_consumer
|
2497
|
+
return wrapper
|
2498
|
+
|
2499
|
+
return decorator
|
2500
|
+
|
2501
|
+
def find_udf_functions(path:str = '.') -> List[UDFDefinition]:
|
2502
|
+
"""
|
2503
|
+
Finds all functions in the specified directory which have the 'omnata_udf' decorator applied
|
2504
|
+
"""
|
2505
|
+
# Get the current directory's absolute path
|
2506
|
+
current_dir = os.path.abspath(path)
|
2507
|
+
|
2508
|
+
# List to hold the classes that match the attribute
|
2509
|
+
matching_classes = []
|
2510
|
+
|
2511
|
+
# Iterate over all modules in the current directory
|
2512
|
+
for _, module_name, _ in pkgutil.iter_modules([current_dir]):
|
2513
|
+
# Import the module
|
2514
|
+
module = importlib.import_module(module_name)
|
2515
|
+
|
2516
|
+
# Iterate over all members of the module
|
2517
|
+
for name, obj in inspect.getmembers(module, inspect.isfunction):
|
2518
|
+
# Check if the class has the specified attribute
|
2519
|
+
if hasattr(obj, '_is_omnata_udf'):
|
2520
|
+
matching_classes.append(UDFDefinition(
|
2521
|
+
name=obj._omnata_udf_name,
|
2522
|
+
language='python',
|
2523
|
+
runtime_version='3.10',
|
2524
|
+
imports=['/app.zip'],
|
2525
|
+
description=obj._omnata_udf_description,
|
2526
|
+
params=obj._omnata_udf_params,
|
2527
|
+
result_data_type=obj._omnata_udf_result_data_type,
|
2528
|
+
expose_to_consumer=obj._omnata_udf_expose_to_consumer,
|
2529
|
+
packages=[],
|
2530
|
+
handler=obj.__module__+'.'+obj.__name__
|
2531
|
+
))
|
2532
|
+
elif isinstance(obj, UDFDefinition) and cast(UDFDefinition,obj).language == 'java':
|
2533
|
+
matching_classes.append(obj)
|
2534
|
+
|
2535
|
+
return matching_classes
|
@@ -20,6 +20,7 @@ from .configuration import (
|
|
20
20
|
OutboundSyncStrategy,
|
21
21
|
StoredConfigurationValue,
|
22
22
|
StoredMappingValue,
|
23
|
+
get_secrets
|
23
24
|
)
|
24
25
|
from .forms import ConnectionMethod, FormInputField, FormOption
|
25
26
|
from .logging import OmnataPluginLogHandler
|
@@ -71,7 +72,7 @@ class PluginEntrypoint:
|
|
71
72
|
def sync(self, sync_request: Dict):
|
72
73
|
logger.info("Entered sync method")
|
73
74
|
request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
|
74
|
-
connection_secrets =
|
75
|
+
connection_secrets = get_secrets(
|
75
76
|
request.oauth_secret_name, request.other_secrets_name
|
76
77
|
)
|
77
78
|
omnata_log_handler = OmnataPluginLogHandler(
|
@@ -270,7 +271,7 @@ class PluginEntrypoint:
|
|
270
271
|
sync_strategy = normalise_nulls(sync_strategy)
|
271
272
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
272
273
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
273
|
-
connection_secrets =
|
274
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
274
275
|
connection_parameters = TypeAdapter(
|
275
276
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
276
277
|
sync_parameters = TypeAdapter(
|
@@ -324,7 +325,7 @@ class PluginEntrypoint:
|
|
324
325
|
logger.info("Entered list_streams method")
|
325
326
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
326
327
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
327
|
-
connection_secrets =
|
328
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
328
329
|
connection_parameters = TypeAdapter(
|
329
330
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
330
331
|
sync_parameters = TypeAdapter(
|
@@ -411,31 +412,6 @@ class PluginEntrypoint:
|
|
411
412
|
raise e
|
412
413
|
return [e.model_dump() for e in events]
|
413
414
|
|
414
|
-
def get_secrets(
|
415
|
-
self, oauth_secret_name: Optional[str], other_secrets_name: Optional[str]
|
416
|
-
) -> Dict[str, StoredConfigurationValue]:
|
417
|
-
connection_secrets = {}
|
418
|
-
import _snowflake # pylint: disable=import-error, import-outside-toplevel # type: ignore
|
419
|
-
if oauth_secret_name is not None:
|
420
|
-
connection_secrets["access_token"] = StoredConfigurationValue(
|
421
|
-
value=_snowflake.get_oauth_access_token(oauth_secret_name)
|
422
|
-
)
|
423
|
-
if other_secrets_name is not None:
|
424
|
-
try:
|
425
|
-
secret_string_content = _snowflake.get_generic_secret_string(
|
426
|
-
other_secrets_name
|
427
|
-
)
|
428
|
-
if len(secret_string_content) > 2:
|
429
|
-
other_secrets = json.loads(secret_string_content)
|
430
|
-
connection_secrets = {
|
431
|
-
**connection_secrets,
|
432
|
-
**TypeAdapter(Dict[str, StoredConfigurationValue]).validate_python(other_secrets),
|
433
|
-
}
|
434
|
-
except Exception as exception:
|
435
|
-
logger.error(f"Error parsing secrets content for secret {other_secrets_name}: {str(exception)}")
|
436
|
-
raise ValueError(f"Error parsing secrets content: {str(exception)}") from exception
|
437
|
-
return connection_secrets
|
438
|
-
|
439
415
|
def ngrok_post_tunnel_fields(
|
440
416
|
self,
|
441
417
|
connection_method: str,
|
@@ -447,7 +423,7 @@ class PluginEntrypoint:
|
|
447
423
|
logger.info("Entered ngrok_post_tunnel_fields method")
|
448
424
|
oauth_secret_name = normalise_nulls(oauth_secret_name)
|
449
425
|
other_secrets_name = normalise_nulls(other_secrets_name)
|
450
|
-
connection_secrets =
|
426
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
451
427
|
connection_parameters = TypeAdapter(
|
452
428
|
Dict[str, StoredConfigurationValue]).validate_python(connection_parameters)
|
453
429
|
parameters = ConnectionConfigurationParameters(
|
@@ -493,7 +469,7 @@ class PluginEntrypoint:
|
|
493
469
|
):
|
494
470
|
logger.info("Entered connect method")
|
495
471
|
logger.info(f"Connection parameters: {connection_parameters}")
|
496
|
-
connection_secrets =
|
472
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
497
473
|
|
498
474
|
from omnata_plugin_runtime.omnata_plugin import (
|
499
475
|
ConnectionConfigurationParameters,
|
@@ -540,7 +516,7 @@ class PluginEntrypoint:
|
|
540
516
|
oauth_secret_name: Optional[str],
|
541
517
|
other_secrets_name: Optional[str]):
|
542
518
|
logger.info("Entered api_limits method")
|
543
|
-
connection_secrets =
|
519
|
+
connection_secrets = get_secrets(oauth_secret_name, other_secrets_name)
|
544
520
|
from omnata_plugin_runtime.omnata_plugin import (
|
545
521
|
ConnectionConfigurationParameters,
|
546
522
|
)
|
File without changes
|
File without changes
|
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/api.py
RENAMED
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/forms.py
RENAMED
File without changes
|
{omnata_plugin_runtime-0.5.5a127 → omnata_plugin_runtime-0.5.6}/src/omnata_plugin_runtime/logging.py
RENAMED
File without changes
|
File without changes
|