omnata-plugin-runtime 0.8.0a187__py3-none-any.whl → 0.8.0a188__py3-none-any.whl

omnata_plugin_runtime/plugin_entrypoints.py

@@ -36,6 +36,7 @@ from .omnata_plugin import (
 )
 from pydantic import TypeAdapter
 from .rate_limiting import ApiLimits, RateLimitState
+from opentelemetry import trace
 
 IMPORT_DIRECTORY_NAME = "snowflake_import_directory"
 
@@ -50,46 +51,40 @@ class PluginEntrypoint:
         self, plugin_fqn: str, session: Session, module_name: str, class_name: str
     ):
         logger.info(f"Initialising plugin entrypoint for {plugin_fqn}")
-        self._session = session
-        import_dir = sys._xoptions[IMPORT_DIRECTORY_NAME]
-        sys.path.append(os.path.join(import_dir, "app.zip"))
-        module = importlib.import_module(module_name)
-        class_obj = getattr(module, class_name)
-        self._plugin_instance: OmnataPlugin = class_obj()
-        self._plugin_instance._session = session  # pylint: disable=protected-access
-        # logging defaults
-        snowflake_logger = logging.getLogger("snowflake")
-        snowflake_logger.setLevel(logging.WARN) # we don't want snowflake queries being logged by default
-        # the sync engine can tell the plugin to override log level via a session variable
-        if session is not None:
-            try:
-                v = session.sql("select getvariable('LOG_LEVEL_OVERRIDES')").collect()
-                result = v[0][0]
-                if result is not None:
-                    log_level_overrides:Dict[str,str] = json.loads(result)
-                    for logger_name,level in log_level_overrides.items():
-                        logger_override = logging.getLogger(logger_name)
-                        logger_override.setLevel(level)
-                        logger_override.propagate = False
-                        for handler in logger_override.handlers:
-                            handler.setLevel(level)
-            except Exception as e:
-                logger.error(f"Error setting log level overrides: {str(e)}")
+        self.tracer = trace.get_tracer(__name__)
+        with self.tracer.start_as_current_span("plugin_initialization") as span:
+            self._session = session
+            import_dir = sys._xoptions[IMPORT_DIRECTORY_NAME]
+            span.add_event("Adding plugin zip to path")
+            sys.path.append(os.path.join(import_dir, "app.zip"))
+            span.add_event("Importing plugin module")
+            module = importlib.import_module(module_name)
+            class_obj = getattr(module, class_name)
+            self._plugin_instance: OmnataPlugin = class_obj()
+            self._plugin_instance._session = session  # pylint: disable=protected-access
+            # logging defaults
+            snowflake_logger = logging.getLogger("snowflake")
+            snowflake_logger.setLevel(logging.WARN) # we don't want snowflake queries being logged by default
+            # the sync engine can tell the plugin to override log level via a session variable
+            if session is not None:
+                try:
+                    span.add_event("Checking log level overrides")
+                    v = session.sql("select getvariable('LOG_LEVEL_OVERRIDES')").collect()
+                    result = v[0][0]
+                    if result is not None:
+                        log_level_overrides:Dict[str,str] = json.loads(result)
+                        span.add_event("Applying log level overrides",log_level_overrides)
+                        for logger_name,level in log_level_overrides.items():
+                            logger_override = logging.getLogger(logger_name)
+                            logger_override.setLevel(level)
+                            logger_override.propagate = False
+                            for handler in logger_override.handlers:
+                                handler.setLevel(level)
+                except Exception as e:
+                    logger.error(f"Error setting log level overrides: {str(e)}")
         
 
     def sync(self, sync_request: Dict):
-        logger.info("Entered sync method")
-        request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
-        connection_secrets = get_secrets(
-            request.oauth_secret_name, request.other_secrets_name
-        )
-        omnata_log_handler = OmnataPluginLogHandler(
-            session=self._session,
-            sync_id=request.sync_id,
-            sync_branch_id=request.sync_branch_id,
-            connection_id=request.connection_id,
-            sync_run_id=request.run_id,
-        )
         logger.add_extra('omnata.operation', 'sync')
         logger.add_extra('omnata.sync.id', request.sync_id)
         logger.add_extra('omnata.sync.direction', request.sync_direction)
@@ -97,50 +92,67 @@ class PluginEntrypoint:
         logger.add_extra('omnata.sync.run_id', request.run_id)
         logger.add_extra('omnata.sync_branch.id', request.sync_branch_id)
         logger.add_extra('omnata.sync_branch.name', request.sync_branch_name)
+        logger.info("Entered sync method")
+        with self.tracer.start_as_current_span("sync_initialization") as span:
+            span.add_event("Fetching secrets")
 
-        omnata_log_handler.register(
-            request.logging_level, self._plugin_instance.additional_loggers()
-        )
-        # construct some connection parameters for the purpose of getting the api limits
-        connection_parameters = ConnectionConfigurationParameters(
-            connection_method=request.connection_method,
-            connectivity_option=request.connectivity_option,
-            connection_parameters=request.connection_parameters,
-            connection_secrets=connection_secrets
-        )
-        if request.oauth_secret_name is not None:
-            connection_parameters.access_token_secret_name = request.oauth_secret_name
-        all_api_limits = self._plugin_instance.api_limits(connection_parameters)
-        logger.info(
-            f"Default API limits: {json.dumps(to_jsonable_python(all_api_limits))}"
-        )
-        all_api_limits_by_category = {
-            api_limit.endpoint_category: api_limit for api_limit in all_api_limits
-        }
-        all_api_limits_by_category.update(
-            {
-                k: v
-                for k, v in [
-                    (x.endpoint_category, x) for x in request.api_limit_overrides
-                ]
+            request = TypeAdapter(SyncRequestPayload).validate_python(sync_request)
+            connection_secrets = get_secrets(
+                request.oauth_secret_name, request.other_secrets_name
+            )
+            span.add_event("Configuring log handler")
+            omnata_log_handler = OmnataPluginLogHandler(
+                session=self._session,
+                sync_id=request.sync_id,
+                sync_branch_id=request.sync_branch_id,
+                connection_id=request.connection_id,
+                sync_run_id=request.run_id,
+            )
+
+            omnata_log_handler.register(
+                request.logging_level, self._plugin_instance.additional_loggers()
+            )
+            # construct some connection parameters for the purpose of getting the api limits
+            connection_parameters = ConnectionConfigurationParameters(
+                connection_method=request.connection_method,
+                connectivity_option=request.connectivity_option,
+                connection_parameters=request.connection_parameters,
+                connection_secrets=connection_secrets
+            )
+            if request.oauth_secret_name is not None:
+                connection_parameters.access_token_secret_name = request.oauth_secret_name
+            span.add_event("Configuring API Limits")
+            all_api_limits = self._plugin_instance.api_limits(connection_parameters)
+            logger.info(
+                f"Default API limits: {json.dumps(to_jsonable_python(all_api_limits))}"
+            )
+            all_api_limits_by_category = {
+                api_limit.endpoint_category: api_limit for api_limit in all_api_limits
             }
-        )
-        api_limits = list(all_api_limits_by_category.values())
-        return_dict = {}
-        logger.info(
-            f"Rate limits state: {json.dumps(to_jsonable_python(request.rate_limits_state))}"
-        )
-        (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(request.rate_limits_state,request.sync_id, request.sync_branch_name)
-        # if any endpoint categories have no state, give them an empty state
-        for api_limit in api_limits:
-            if api_limit.endpoint_category not in rate_limit_state_all:
-                rate_limit_state_all[api_limit.endpoint_category] = RateLimitState(
-                    wait_until=None, previous_request_timestamps=[]
-                )
-            if api_limit.endpoint_category not in rate_limit_state_this_branch:
-                rate_limit_state_this_branch[api_limit.endpoint_category] = RateLimitState(
-                    wait_until=None, previous_request_timestamps=[]
-                )
+            all_api_limits_by_category.update(
+                {
+                    k: v
+                    for k, v in [
+                        (x.endpoint_category, x) for x in request.api_limit_overrides
+                    ]
+                }
+            )
+            api_limits = list(all_api_limits_by_category.values())
+            return_dict = {}
+            logger.info(
+                f"Rate limits state: {json.dumps(to_jsonable_python(request.rate_limits_state))}"
+            )
+            (rate_limit_state_all, rate_limit_state_this_branch) = RateLimitState.collapse(request.rate_limits_state,request.sync_id, request.sync_branch_name)
+            # if any endpoint categories have no state, give them an empty state
+            for api_limit in api_limits:
+                if api_limit.endpoint_category not in rate_limit_state_all:
+                    rate_limit_state_all[api_limit.endpoint_category] = RateLimitState(
+                        wait_until=None, previous_request_timestamps=[]
+                    )
+                if api_limit.endpoint_category not in rate_limit_state_this_branch:
+                    rate_limit_state_this_branch[api_limit.endpoint_category] = RateLimitState(
+                        wait_until=None, previous_request_timestamps=[]
+                    )
 
         if request.sync_direction == "outbound":
             parameters = OutboundSyncConfigurationParameters(
@@ -174,11 +186,13 @@ class PluginEntrypoint:
             )
             try:
                 self._plugin_instance._configuration_parameters = parameters
-                with HttpRateLimiting(outbound_sync_request, parameters):
-                    self._plugin_instance.sync_outbound(parameters, outbound_sync_request)
+                with self.tracer.start_as_current_span("sync_execution") as span:
+                    with HttpRateLimiting(outbound_sync_request, parameters):
+                        self._plugin_instance.sync_outbound(parameters, outbound_sync_request)
                 if self._plugin_instance.disable_background_workers is False:
-                    outbound_sync_request.apply_results_queue()
-                    outbound_sync_request.apply_rate_limit_state()
+                    with self.tracer.start_as_current_span("results_finalization") as span:
+                        outbound_sync_request.apply_results_queue()
+                        outbound_sync_request.apply_rate_limit_state()
                 if outbound_sync_request.deadline_reached:
                     # if we actually hit the deadline, this is flagged by the cancellation checking worker and the cancellation
                     # token is set. We throw it here as an error since that's currently how it flows back to the engine with a DELAYED state
@@ -232,19 +246,21 @@ class PluginEntrypoint:
                 inbound_sync_request.update_activity("Invoking plugin")
                 logger.info(f"inbound sync request: {inbound_sync_request}")
                 # plugin_instance._inbound_sync_request = outbound_sync_request
-                with HttpRateLimiting(inbound_sync_request, parameters):
-                    self._plugin_instance.sync_inbound(parameters, inbound_sync_request)
+                with self.tracer.start_as_current_span("sync_execution") as span:
+                    with HttpRateLimiting(inbound_sync_request, parameters):
+                        self._plugin_instance.sync_inbound(parameters, inbound_sync_request)
                 logger.info("Finished invoking plugin")
                 if self._plugin_instance.disable_background_workers is False:
-                    inbound_sync_request.update_activity("Staging remaining records")
-                    logger.info("Calling apply_results_queue")
-                    inbound_sync_request.apply_results_queue()
-                    try:
-                        # this is not critical, we wouldn't fail the sync over rate limit usage capture
-                        logger.info("Calling apply_rate_limit_state")
-                        inbound_sync_request.apply_rate_limit_state()
-                    except Exception as e:
-                        logger.error(f"Error applying rate limit state: {str(e)}")
+                    with self.tracer.start_as_current_span("results_finalization") as span:
+                        inbound_sync_request.update_activity("Staging remaining records")
+                        logger.info("Calling apply_results_queue")
+                        inbound_sync_request.apply_results_queue()
+                        try:
+                            # this is not critical, we wouldn't fail the sync over rate limit usage capture
+                            logger.info("Calling apply_rate_limit_state")
+                            inbound_sync_request.apply_rate_limit_state()
+                        except Exception as e:
+                            logger.error(f"Error applying rate limit state: {str(e)}")
                     # here we used to do a final inbound_sync_request.apply_progress_updates(ignore_errors=False)
                     # but it was erroring too much since there was usually a lot of DDL activity on the Snowflake side
                     # so instead, we'll provide a final progress update via a return value from the proc

{omnata_plugin_runtime-0.8.0a187.dist-info → omnata_plugin_runtime-0.8.0a188.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.8.0a187
+Version: 0.8.0a188
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.8.0a187.dist-info → omnata_plugin_runtime-0.8.0a188.dist-info}/RECORD

@@ -4,9 +4,9 @@ omnata_plugin_runtime/configuration.py,sha256=5M7dmQu9BO2msES9Foa8fOKfWui7iPSExA
 omnata_plugin_runtime/forms.py,sha256=ueodN2GIMS5N9fqebpY4uNGJnjEb9HcuaVQVfWH-cGg,19838
 omnata_plugin_runtime/logging.py,sha256=u_Bo2v4jS3C_2E_Y8a7yfZZcIP-h5Mak_FPnFHUwFbU,4378
 omnata_plugin_runtime/omnata_plugin.py,sha256=aggjb_CTTjhgqjS8CHPOm4ENU0jNcYoT6LC8yI1IeF4,130048
-omnata_plugin_runtime/plugin_entrypoints.py,sha256=LM4FHMW0N3N13oxsUqkSEbUhH95uFw-yNIJ5kNGDgWE,30633
+omnata_plugin_runtime/plugin_entrypoints.py,sha256=z1NJpWvEj5TizCEU8YLnO5cWmeU8iSEsMK9CQaL47RA,32021
 omnata_plugin_runtime/rate_limiting.py,sha256=JKtyz8mA9D0FSZgQplPusmk2rVclcjkwtE59fQQrQ_I,25610
-omnata_plugin_runtime-0.8.0a187.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
-omnata_plugin_runtime-0.8.0a187.dist-info/METADATA,sha256=uVZoK32EmYnNrEAi7mDHt6x9iVZ2IvrBXWBqpmmbNeo,2148
-omnata_plugin_runtime-0.8.0a187.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
-omnata_plugin_runtime-0.8.0a187.dist-info/RECORD,,
+omnata_plugin_runtime-0.8.0a188.dist-info/LICENSE,sha256=rGaMQG3R3F5-JGDp_-rlMKpDIkg5n0SI4kctTk8eZSI,56
+omnata_plugin_runtime-0.8.0a188.dist-info/METADATA,sha256=OrGtX56IyJCjpFTsxkVu0IZI3Z9zJ5aHSzpgI0Mg3Bk,2148
+omnata_plugin_runtime-0.8.0a188.dist-info/WHEEL,sha256=Nq82e9rUAnEjt98J6MlVmMCZb-t9cYE2Ir1kpBmnWfs,88
+omnata_plugin_runtime-0.8.0a188.dist-info/RECORD,,