omnata-plugin-runtime 0.1.0__py3-none-any.whl

omnata_plugin_runtime/plugin_entrypoints.py

@@ -0,0 +1,286 @@
+
+import json
+import sys
+import os
+import importlib
+import datetime
+from logging import getLogger
+from typing import Dict, List, Union, Optional
+from omnata_plugin_runtime.forms import ConnectionMethod
+from omnata_plugin_runtime.logging import OmnataPluginLogHandler
+from omnata_plugin_runtime.rate_limiting import ApiLimits
+from pydantic import BaseModel, parse_obj_as # pylint: disable=no-name-in-module
+from pydantic.json import pydantic_encoder as pydantic_json_encoder # pylint:disable=no-name-in-module
+from snowflake.snowpark import Session
+from omnata_plugin_runtime.omnata_plugin import OmnataPlugin, HttpRateLimiting, OutboundSyncRequest, InboundSyncRequest
+from omnata_plugin_runtime.configuration import OutboundSyncConfigurationParameters,SyncDirection,InboundSyncConfigurationParameters, OutboundSyncStrategy, StoredConfigurationValue,SyncConfigurationParameters,InboundSyncStreamsConfiguration, StreamConfiguration, StoredStreamConfiguration, SyncConfigurationParameters, StoredMappingValue
+from omnata_plugin_runtime.rate_limiting import ApiLimits, RetryLaterException, RateLimitState, RequestRateLimit
+from omnata_plugin_runtime.api import ApplyPayload
+logger = getLogger(__name__)
+# this is the new API for secrets access (https://docs.snowflake.com/en/LIMITEDACCESS/secret-api-reference)
+import _snowflake # pylint: disable=import-error
+IMPORT_DIRECTORY_NAME = "snowflake_import_directory"
+
+class PluginEntrypoint():
+    """
+    This class gives each plugin's stored procs an initial point of contact.
+    It will only work within Snowflake because it uses the _snowflake module.
+    """
+    def __init__(self,
+                 plugin_fqn:str,
+                 session:Session,
+                 module_name:str,
+                 class_name:str):
+        logger.info('Initialising plugin entrypoint')
+        self._session = session
+        import_dir = sys._xoptions[IMPORT_DIRECTORY_NAME]
+        sys.path.append(os.path.join(import_dir,'app.zip'))
+        module = importlib.import_module(module_name)
+        class_obj = getattr(module, class_name)
+        self._plugin_instance:OmnataPlugin = class_obj()
+
+    def apply(self,apply_request:Dict):
+        logger.info('Entered apply method')
+        request = parse_obj_as(ApplyPayload,apply_request)
+        connection_secrets = self.get_secrets(request.oauth_secret_name,request.other_secrets_name)
+        omnata_log_handler = OmnataPluginLogHandler(session=self._session,
+                                                sync_id=request.sync_id,
+                                                sync_branch_id=request.sync_branch_id,
+                                                connection_id=request.connection_id,
+                                                sync_run_id=request.run_id)
+        omnata_log_handler.register(request.logging_level,self._plugin_instance.additional_loggers())
+        # construct some generic parameters for the purpose of getting the api limits
+        base_parameters = SyncConfigurationParameters(request.connection_method,
+                            connection_parameters=request.connection_parameters,
+                            connection_secrets=connection_secrets,
+                            sync_parameters=request.sync_parameters,
+                            current_form_parameters={})
+        all_api_limits = self._plugin_instance.api_limits(base_parameters)
+        logger.info(f"Default API limits: {json.dumps(all_api_limits, default=pydantic_json_encoder)}")
+        all_api_limits_by_category = {api_limit.endpoint_category:api_limit for api_limit in all_api_limits}
+        all_api_limits_by_category.update({k:v for k,v in [(x.endpoint_category,x) for x in request.api_limit_overrides]})
+        api_limits = list(all_api_limits_by_category.values())
+        # if any endpoint categories have no state, give them an empty state
+        for api_limit in api_limits:
+            if api_limit.endpoint_category not in request.rate_limits_state:
+                request.rate_limits_state[api_limit.endpoint_category] = RateLimitState()
+        logger.info(f"Rate limits state: {json.dumps(request.rate_limits_state, default=pydantic_json_encoder)}")
+        if request.sync_direction=='outbound':
+            parameters = OutboundSyncConfigurationParameters(
+                connection_method=request.connection_method,
+                connection_parameters=request.connection_parameters,
+                connection_secrets=connection_secrets,
+                sync_parameters=request.sync_parameters,
+                current_form_parameters={},
+                sync_strategy=request.sync_strategy,
+                field_mappings=request.field_mappings,
+            )
+
+            outbound_sync_request = OutboundSyncRequest(run_id=request.run_id,
+                                            session=self._session,
+                                            source_app_name=request.source_app_name,
+                                            results_schema_name=request.results_schema_name,
+                                            results_table_name=request.results_table_name,
+                                            plugin_instance=self._plugin_instance,
+                                            api_limits=request.api_limits,
+                                            rate_limit_state=request.rate_limits_state,
+                                            run_deadline=datetime.datetime.now() + datetime.timedelta(hours=4),
+                                            development_mode=False
+                                            )
+            
+            with HttpRateLimiting(outbound_sync_request,parameters):
+                combined_results = self._plugin_instance.sync_outbound(parameters,outbound_sync_request)
+                if combined_results is not None:
+                    outbound_sync_request.enqueue_results(combined_results)
+                    outbound_sync_request.apply_results_queue()
+        
+        elif request.sync_direction=='inbound':
+            logger.info("Running inbound sync")
+            parameters = InboundSyncConfigurationParameters(
+                connection_method=request.connection_method,
+                connection_parameters=request.connection_parameters,
+                connection_secrets=connection_secrets,
+                sync_parameters=request.sync_parameters,
+                current_form_parameters={})
+            
+            # build streams object from parameters
+            streams_list:List[StoredStreamConfiguration] = []
+            streams_list = streams_list + list(request.streams_configuration.included_streams.values())
+            
+            # if new streams are included, we need to fetch the list first to find them
+            if request.streams_configuration.include_new_streams:
+                # we have to invoke the inbound_configuration_form to get the StreamLister, as the list
+                # of streams may vary based on the sync parameters
+                form = self._plugin_instance.inbound_configuration_form(parameters)
+                if form.stream_lister is None:
+                    logger.info("No stream lister defined, skipping new stream detection")
+                else:
+                    all_streams:List[StreamConfiguration] = getattr(self._plugin_instance,form.stream_lister.source_function)(parameters)
+                    for s in all_streams:
+                        if s.stream_name not in request.streams_configuration.included_streams and \
+                                    s.stream_name not in request.streams_configuration.excluded_streams:
+                            if request.streams_configuration.new_stream_sync_strategy not in s.supported_sync_strategies:
+                                raise ValueError(f"New object {s.stream_name} was found, but does not support the defined sync strategy {request.streams_configuration}")
+                            
+                            new_stream = StoredStreamConfiguration(stream_name=s.stream_name,
+                                                                    cursor_field=s.source_defined_cursor,
+                                                                    primary_key_field=s.source_defined_primary_key,
+                                                                    latest_state={},
+                                                                    storage_behaviour=request.streams_configuration.new_stream_storage_behaviour,
+                                                                    stream=s,
+                                                                    sync_strategy=request.streams_configuration.new_stream_sync_strategy
+                                                                    )
+                            streams_list.append(new_stream)
+            
+            for stream in streams_list:
+                if stream.stream_name in request.latest_stream_state:
+                    stream.latest_state = request.request.latest_stream_state[stream.stream_name]
+                    logger.info(f"Updating stream state for {stream.stream_name}: {stream.latest_state}")
+                else:
+                    logger.info(f"Existing stream state for {stream.stream_name} not found")
+            # update the sync run to show the total number of streams to check
+            # TODO: make this a function call
+            #sync_run.INBOUND_STREAM_TOTAL_COUNTS = {stream.stream_name:0 for stream in streams_list}
+            #sync_run.update_object()
+            logger.info(f"streams list: {streams_list}")
+            logger.info(f"streams config: {request.streams_configuration}")
+            inbound_sync_request = InboundSyncRequest(run_id=request.run_id,
+                                            session=self._session,
+                                            source_app_name=request.source_app_name,
+                                            results_schema_name=request.results_schema_name,
+                                            results_table_name=request.results_table_name,
+                                            plugin_instance=self._plugin_instance,
+                                            api_limits=request.api_limits,
+                                            rate_limit_state=request.rate_limits_state,
+                                            run_deadline=datetime.datetime.now() + datetime.timedelta(hours=4),
+                                            development_mode=False,
+                                            streams=streams_list
+                                            )
+
+            inbound_sync_request.update_activity(f"Invoking plugin")
+            logger.info(f"inbound sync request: {inbound_sync_request}")
+            #plugin_instance._inbound_sync_request = outbound_sync_request
+            with HttpRateLimiting(inbound_sync_request,parameters):
+                combined_results = self._plugin_instance.sync_inbound(parameters,inbound_sync_request)
+            logger.info(f"Finished invoking plugin")
+            inbound_sync_request.update_activity(f"Staging remaining records")
+            logger.info(f"Calling apply_results_queue")
+            inbound_sync_request.apply_results_queue()
+            omnata_log_handler.flush()
+            # we need to calculate counts for:
+            # CHANGED_COUNT by counting up the records in INBOUND_STREAM_RECORD_COUNTS
+        logger.info('Finished applying records')
+        omnata_log_handler.flush()
+        omnata_log_handler.unregister()
+        return {
+            "streams": dict(streams_list)
+        }
+
+    def configuration_form(self,
+                           connection_method:str,
+                           connection_parameters:Dict,
+                           oauth_secret_name:Optional[str],
+                           other_secrets_name:Optional[str],
+                           sync_direction:str,
+                           sync_strategy:Dict,
+                           function_name:str,
+                           sync_parameters:Dict,
+                           current_form_parameters:Optional[Dict]):
+        logger.info(f'Entered configuration_form method')
+        sync_strategy = normalise_nulls(sync_strategy)
+        oauth_secret_name = normalise_nulls(oauth_secret_name)
+        other_secrets_name = normalise_nulls(other_secrets_name)
+        connection_secrets=self.get_secrets(oauth_secret_name,other_secrets_name)
+        connection_parameters = parse_obj_as(Dict[str,StoredConfigurationValue],connection_parameters)
+        sync_parameters = parse_obj_as(Dict[str,StoredConfigurationValue],sync_parameters)
+        form_parameters = None
+        if current_form_parameters is not None:
+            form_parameters = parse_obj_as(Dict[str,StoredConfigurationValue],current_form_parameters)
+        if sync_direction=='outbound':
+            sync_strat = OutboundSyncStrategy.parse_obj(sync_strategy)
+            parameters = OutboundSyncConfigurationParameters(connection_parameters=connection_parameters,
+                                                        connection_secrets=connection_secrets,
+                                                        sync_strategy=sync_strat,
+                                                        sync_parameters=sync_parameters,
+                                                        connection_method=connection_method,
+                                                        current_form_parameters=form_parameters)
+        elif sync_direction=='inbound':
+            parameters = InboundSyncConfigurationParameters(connection_parameters=connection_parameters,
+                                                        connection_secrets=connection_secrets,
+                                                        sync_parameters=sync_parameters,
+                                                        connection_method=connection_method,
+                                                        current_form_parameters=form_parameters)
+        else:
+            raise ValueError(f'Unknown direction {sync_direction}')
+        the_function = getattr(self._plugin_instance, function_name or f"{sync_direction}_configuration_form")
+        script_result = the_function(parameters)
+        if isinstance(script_result,BaseModel):
+            script_result = script_result.dict()
+        elif isinstance(script_result,List):
+            if len(script_result) > 0 and isinstance(script_result[0],BaseModel):
+                script_result = [r.dict() for r in script_result]
+        return script_result
+        
+    def connection_form(self):
+        try:
+            logger.info('Entered connection_form method')
+            form:List[ConnectionMethod] = self._plugin_instance.connection_form()
+            return [f.dict() for f in form]
+        except Exception as exception:
+            logger.error(str(exception),exc_info=True,stack_info=True)
+            return {
+                "success": False,
+                "error": str(exception)
+            }
+
+    def get_secrets(self,oauth_secret_name:Optional[str],other_secrets_name:Optional[str]) -> Dict[str,StoredConfigurationValue]:
+        connection_secrets={}
+        if oauth_secret_name is not None:
+            connection_secrets['access_token']= StoredConfigurationValue(value=_snowflake.get_oauth_access_token(oauth_secret_name))
+        if other_secrets_name is not None:
+            try:
+                secret_string_content = _snowflake.get_generic_secret_string(other_secrets_name)
+                other_secrets = json.loads(secret_string_content)
+            except Exception as exception:
+                logger.error(f'Error parsing secrets content: {str(exception)}')
+                raise ValueError(f'Error parsing secrets content: {str(exception)}')
+            connection_secrets = parse_obj_as(Dict[str,StoredConfigurationValue],other_secrets)
+        return connection_secrets
+    
+    def connect(self,method,connection_parameters:Dict,network_rule_name:str,oauth_secret_name:Optional[str],other_secrets_name:Optional[str]):
+        logger.info('Entered connect method')
+        logger.info(f'Connection parameters: {connection_parameters}')
+        connection_secrets = self.get_secrets(oauth_secret_name,other_secrets_name)
+        
+        from omnata_plugin_runtime.omnata_plugin import ConnectionConfigurationParameters
+        connect_response = self._plugin_instance.connect(ConnectionConfigurationParameters(connection_method=method,
+                                                                connection_parameters=parse_obj_as(Dict[str,StoredConfigurationValue],connection_parameters),
+                                                                connection_secrets=parse_obj_as(Dict[str,StoredConfigurationValue],connection_secrets)))
+        # the connect method can also return more network addresses. If so, we need to update the 
+        # network rule associated with the external access integration
+        if connect_response.network_addresses is not None:
+            existing_rule_result = self._session.sql(f'desc network rule {network_rule_name}').collect()
+            rule_values:List[str] = existing_rule_result[0].value_list.split(',')
+            for network_address in connect_response.network_addresses:
+                if network_address not in rule_values:
+                    rule_values.append(network_address)
+            rule_values_string = ','.join([f"'{value}'" for value in rule_values])
+            self._session.sql(f'alter network rule {network_rule_name} set value_list = ({rule_values_string})').collect()
+
+        return connect_response.dict()
+
+    def api_limits(self):
+        logger.info('Entered api_limits method')
+        response:List[ApiLimits] = self._plugin_instance.api_limits(None)
+        return [api_limit.dict() for api_limit in response]
+
+def normalise_nulls(obj):
+    """
+    If an object came through a SQL interface with a null value, we convert it to a regular None here
+    """
+    if type(obj).__name__=='sqlNullWrapper':
+        return None
+    # handle a bunch of objects being given at once
+    if type(obj).__name__=='list':
+        return [normalise_nulls(x) for x in obj]
+    return obj

omnata_plugin_runtime/rate_limiting.py

@@ -0,0 +1,232 @@
+# it's not the 1980s anymore
+# pylint: disable=line-too-long,multiple-imports,logging-fstring-interpolation
+"""
+Contains functionality for limiting http requests made by Omnata plugins
+"""
+from __future__ import annotations
+from pydantic import Field
+from typing import List, Literal, Optional
+from logging import getLogger
+from email.utils import parsedate_to_datetime
+from .configuration import SubscriptableBaseModel
+import datetime
+import threading
+import requests
+import re
+logger = getLogger(__name__)
+
+TimeUnitType = Literal['second','minute','hour','day']
+
+HttpMethodType = Literal['GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH']
+
+class HttpRequestMatcher(SubscriptableBaseModel):
+    """
+    A class used to match an HTTP request
+    """
+    http_methods:List[HttpMethodType]
+    url_regex:str
+
+    @classmethod
+    def match_all(cls):
+        """
+        A HttpRequestMatcher which will match all requests.
+        """
+        return cls(http_methods=['GET','HEAD','POST','PUT','DELETE','CONNECT','OPTIONS','TRACE','PATCH'],
+                    url_regex='.*')
+
+class ApiLimits(SubscriptableBaseModel):
+    """
+    Encapsulates the constraints imposed by an app's APIs
+    """
+    endpoint_category:str = Field(
+        'All endpoints',
+        description='the name of the API category (e.g. "Data loading endpoints")',
+    )
+    request_matchers:List[HttpRequestMatcher]=Field(
+        [HttpRequestMatcher.match_all()],
+        description="a list of request matchers. If None is provided, all requests will be matched",
+    )
+    request_rates:List[RequestRateLimit]=Field(
+        None,
+        description="imposes time delays between requests to stay under a defined rate limit",
+    )
+
+    def request_matches(self,method:HttpMethodType,url:str):
+        """
+        Given the request matchers that exist, determines whether the provided HTTP method and url is a match
+        """
+        for request_matcher in self.request_matchers:
+            if method in request_matcher.http_methods and re.search(request_matcher.url_regex,url):
+                return True
+        return False
+
+    @classmethod
+    def apply_overrides(cls, default_api_limits:List[ApiLimits],overridden_values:List[ApiLimits]) -> List[ApiLimits]:
+        """
+        Takes a list of default api limits, and replaces them with any overridden values
+        """
+        if overridden_values is None or len(overridden_values)==0:
+            return default_api_limits
+        overrides_keyed = {l.construct_from_variant:l for l in overridden_values}
+        for api_limit in default_api_limits:
+            if api_limit.endpoint_category in overrides_keyed.keys():
+                api_limit.request_rates = overrides_keyed[api_limit.endpoint_category]
+        return default_api_limits
+
+    @classmethod
+    def request_match(cls, all_api_limits:List[ApiLimits],method:HttpMethodType,url:str) -> Optional[ApiLimits]:
+        """
+        Given a set of defined API limits, return the first one that matches, or None if none of them match
+        """
+        for api_limits in all_api_limits:
+            if api_limits.request_matches(method,url):
+                return api_limits
+        return None
+    
+    def calculate_wait(self,rate_limit_state:RateLimitState) -> datetime.datetime:
+        """
+        Based on the configured wait limits, given a sorted list of previous requests (newest to oldest),
+        determine when the next request is allowed to occur.
+        Each rate limit is a number of requests over a time window.
+        Examples:
+        If the rate limit is 5 requests every 10 seconds, we:
+         - determine the timestamp of the 5th most recent request
+         - add 10 seconds to that timestamp
+        The resulting timestamp is when the next request can be made (if it's in the past, it can be done immediately)
+        If multiple rate limits exist, the maximum timestamp is used (i.e. the most restrictive rate limit applies)
+        """
+        logger.info(f"calculating wait time, given previous requests as {rate_limit_state.previous_request_timestamps}")
+        if self.request_rates is None:
+            return datetime.datetime.utcnow()
+        longest_wait = datetime.datetime.utcnow()
+        if rate_limit_state.wait_until is not None and rate_limit_state.wait_until > longest_wait:
+            longest_wait = rate_limit_state.wait_until
+        for request_rate in self.request_rates:
+            if len(rate_limit_state.previous_request_timestamps) > 0:
+                request_index = request_rate.request_count - 1
+                if len(rate_limit_state.previous_request_timestamps) < request_index:
+                    request_index = len(rate_limit_state.previous_request_timestamps)-1
+                timestamp_at_horizon = rate_limit_state.previous_request_timestamps[request_index]
+                next_allowed_request = timestamp_at_horizon + datetime.timedelta(seconds=request_rate.number_of_seconds())
+                if next_allowed_request > longest_wait:
+                    longest_wait = next_allowed_request
+
+        return longest_wait
+
+class RateLimitState(SubscriptableBaseModel):
+    """
+    Encapsulates the rate limiting state of an endpoint category 
+    for a particular connection (as opposed to configuration)
+    """
+    wait_until:Optional[datetime.datetime]=Field(
+        None,
+        description="Providing a value here means that no requests should occur until a specific moment in the future",
+    )
+    previous_request_timestamps:Optional[List[datetime.datetime]]=Field(
+        [],
+        description="A list of timestamps where previous requests have been made, used to calculate the next request time",
+    )
+    _request_timestamps_lock = threading.Lock()
+    
+    def register_http_request(self):
+        """
+        Registers a request as having just occurred, for rate limiting purposes.
+        You only need to use this if your HTTP requests are not automatically being
+        registered, which happens if http.client.HTTPConnection is not being used.
+        """
+        with self._request_timestamps_lock:
+            append_time = datetime.datetime.utcnow()
+            self.previous_request_timestamps.insert(0,append_time)
+    
+    def prune_history(self,request_rates:List[RequestRateLimit]=None):
+        """
+        When we store the request history, it doesn't make sense to go back indefinitely.
+        We only need the requests which fall within the longest rate limiting window
+
+        """
+        longest_window_seconds = max([rate.number_of_seconds() for rate in request_rates])
+        irrelevance_horizon = datetime.datetime.now() - datetime.timedelta(seconds=longest_window_seconds)
+        self.previous_request_timestamps = [ts for ts in self.previous_request_timestamps if ts > irrelevance_horizon]
+
+class RequestRateLimit(SubscriptableBaseModel):
+    """
+    Request rate limits
+	Defined as a request count, time unit and number of units e.g. (1,"second",5) = 1 request per 5 seconds, or (100, "minute", 15) = 100 requests per 15 minutes
+    """
+    request_count:int
+    time_unit:TimeUnitType
+    unit_count:int
+
+    def number_of_seconds(self):
+        """
+        Converts the time_unit and unit_count to a number of seconds.
+        E.g. 5 minutes = 300
+        2 hours = 7200
+        """
+        if self.time_unit=='second':
+            return self.unit_count
+        elif self.time_unit=='minute':
+            return self.unit_count*60
+        elif self.time_unit=='hour':
+            return self.unit_count*3600
+        elif self.time_unit=='day':
+            return self.unit_count*86400
+        else:
+            raise ValueError(f"Unknown time unit: {self.time_unit}")
+    
+    def to_description(self) -> str:
+        """Returns a readable description of this limit.
+        For example:
+        "1 request per minute"
+        "5 requests per 2 seconds"
+
+        Returns:
+            str: the description as described above
+        """
+        return str(self.request_count) + " " + \
+             "request" + ("s" if self.request_count > 1 else "") + \
+             " per " + \
+            (self.time_unit if self.unit_count==1 else f"{self.unit_count} {self.time_unit}s")
+
+class RetryLaterException(Exception):
+    """
+    Exception raised when the app has notified that rate limits are exceeded.
+    Throwing this during record apply imposes a temporary extra API constraint that
+    we need to wait until a future date before more requests are made.
+
+    """
+
+    def __init__(self, future_datetime:datetime.datetime):
+        self.future_datetime = future_datetime
+        message = "Remote system wants us to retry later"
+        self.message = message
+        super().__init__(self.message)
+
+class InterruptedWhileWaitingException(Exception):
+    """
+    Indicates that while waiting for rate limiting to expire, the sync was interrupted
+    """
+
+def too_many_requests_hook(fallback_future_datetime:datetime.datetime = datetime.datetime.utcnow() + datetime.timedelta(hours=24)):
+    """
+    A Requests hook which raises a RetryLaterException if an HTTP 429 response is returned.
+    Examines the Retry-After header (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After)
+    to determine the appropriate future datetime to retry.
+    If that isn't available, it falls back to fallback_future_datetime.
+
+    """
+    def hook(resp:requests.Response, *args, **kwargs):
+        """
+        The actual hook implementation
+        """
+        if resp.status_code==429:
+            if 'Retry-After' in resp.headers:
+                retry_after:str = resp.headers['Retry-After']
+                if retry_after.isnumeric():
+                    raise RetryLaterException(future_datetime=datetime.datetime.utcnow() + datetime.timedelta(seconds=int(retry_after)))
+                retry_date = parsedate_to_datetime(retry_after)
+                raise RetryLaterException(future_datetime=retry_date)
+            raise RetryLaterException(future_datetime=fallback_future_datetime)
+    return hook
+
+ApiLimits.update_forward_refs()

omnata_plugin_runtime/record_transformer.py

@@ -0,0 +1,50 @@
+
+from typing import Dict, List
+from pydantic import parse_obj_as # pylint:disable=no-name-in-module
+from logging import getLogger
+from .configuration import StoredConfigurationValue, StoredMappingValue, StoredFieldMapping
+logger = getLogger(__name__)
+
+class RecordTransformerResult(dict):
+    """
+    The result of transforming an individual record
+    """
+    def __init__(self, success:bool,errors:List[str] = None,transformed_record:dict = None):
+        """
+        Constructs a RecordTransformerResult
+        """
+        if success:
+            dict.__init__(self, success=success,transformed_record=transformed_record)
+        else:
+            dict.__init__(self, success=success,errors=errors)
+
+class RecordTransformer():
+    """
+    The default record transformer, used to convert Snowflake records into an app-specific representation
+    using the configured field mappings, and optionally perform data validation so that source failures occur
+    """
+    def transform_record(self,source_metadata:dict,sync_parameters:Dict[str, StoredConfigurationValue],field_mappings:StoredMappingValue,source_record:dict) -> RecordTransformerResult:
+        """
+        Default transformer.
+        When the visual field mapper is used, simply picks out the columns and renames them.
+        When the jinja mapper is used, copies the template into the output
+        """
+        transformed_record = {}
+        errors = []
+        if field_mappings.mapper_type == 'jinja_template':
+            transformed_record['jinja_template'] = field_mappings.jinja_template
+            transformed_record['source_record'] = source_record
+        elif field_mappings.mapper_type == 'field_mapping_selector':
+            parsed_field_mappings:List[StoredFieldMapping] = parse_obj_as(List[StoredFieldMapping],field_mappings.field_mappings)
+            for field_mapping in parsed_field_mappings:
+                if field_mapping.source_column not in source_record:
+                    errors.append(f"Column '{field_mapping.source_column}' not found in record")
+                else:
+                    transformed_record[field_mapping.app_field] = source_record[field_mapping.source_column]
+        else:
+            errors.append(f"Unrecognised mapper type: {field_mappings.mapper_type}")
+        logger.debug(f"Transformed record: {transformed_record}")
+        if len(errors) > 0:
+            logger.info(f"Record transformer errors: {errors}")
+            return dict(RecordTransformerResult(success=False, errors=errors))
+        return dict(RecordTransformerResult(success=True, transformed_record=transformed_record))