omnata-plugin-runtime 0.8.0a192__tar.gz → 0.8.0a194__tar.gz

{omnata_plugin_runtime-0.8.0a192 → omnata_plugin_runtime-0.8.0a194}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: omnata-plugin-runtime
-Version: 0.8.0a192
+Version: 0.8.0a194
 Summary: Classes and common runtime components for building and running Omnata Plugins
 Author: James Weakley
 Author-email: james.weakley@omnata.com

{omnata_plugin_runtime-0.8.0a192 → omnata_plugin_runtime-0.8.0a194}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.8.0-a192"
+version = "0.8.0-a194"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.8.0a192 → omnata_plugin_runtime-0.8.0a194}/src/omnata_plugin_runtime/logging.py

@@ -8,6 +8,7 @@ from logging import Logger, getLogger
 from typing import Dict, List, Optional
 from snowflake.snowpark import Session
 from pydantic import ValidationError
+from snowflake import telemetry
 from opentelemetry import trace
 
 
@@ -34,7 +35,7 @@ class CustomLoggerAdapter(logging.LoggerAdapter):
         kwargs["extra"] = extra
         return msg, kwargs
 
-logger = CustomLoggerAdapter(getLogger("omnata"), {})
+logger = CustomLoggerAdapter(getLogger("omnata_plugin"), {})
 
 def log_exception(exception: Exception, logger_instance: Logger):
     """

{omnata_plugin_runtime-0.8.0a192 → omnata_plugin_runtime-0.8.0a194}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -47,7 +47,7 @@ from snowflake.snowpark import Session
 from snowflake.snowpark.functions import col
 from tenacity import Retrying, stop_after_attempt, wait_fixed, retry_if_exception_message
 
-from .logging import OmnataPluginLogHandler
+from .logging import OmnataPluginLogHandler, logger
 
 from .api import (
     PluginMessage,
@@ -87,8 +87,9 @@ from .rate_limiting import (
     RateLimitState,
     RateLimitedSession
 )
+from opentelemetry import trace
+tracer = trace.get_tracer('omnata_plugin_runtime')
 
-logger = getLogger(__name__)
 SortDirectionType = Literal["asc", "desc"]
 
 
@@ -810,22 +811,23 @@ class OutboundSyncRequest(SyncRequest):
         logger.debug("applying results to table")
         # use a random table name with a random string to avoid collisions
         with self._snowflake_query_lock:
-            for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
-                with attempt:
-                    success, nchunks, nrows, _ = write_pandas(
-                        conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                        df=self._preprocess_results_dataframe(results_df),
-                        quote_identifiers=False,
-                        table_name=self._full_results_table_name,
-                        auto_create_table=False
-                    )
-                    if not success:
-                        raise ValueError(
-                            f"Failed to write results to table {self._full_results_table_name}"
+            with tracer.start_as_current_span("apply_results"):
+                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
+                    with attempt:
+                        success, nchunks, nrows, _ = write_pandas(
+                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                            df=self._preprocess_results_dataframe(results_df),
+                            quote_identifiers=False,
+                            table_name=self._full_results_table_name,
+                            auto_create_table=False
+                        )
+                        if not success:
+                            raise ValueError(
+                                f"Failed to write results to table {self._full_results_table_name}"
+                            )
+                        logger.debug(
+                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
                         )
-                    logger.debug(
-                        f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
-                    )
 
     def __dataframe_wrapper(
         self, data_frame: pandas.DataFrame, render_jinja: bool = True
@@ -1465,36 +1467,37 @@ class InboundSyncRequest(SyncRequest):
         """
         if len(results_df) > 0:
             with self._snowflake_query_lock:
-                for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
-                    with attempt:
-                        logger.debug(
-                            f"Applying {len(results_df)} results to {self._full_results_table_name}"
-                        )
-                        # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
-                        success, nchunks, nrows, _ = write_pandas(
-                            conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
-                            df=results_df,
-                            table_name=self._full_results_table_name,
-                            quote_identifiers=False,  # already done in get_temp_table_name
-                            # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
-                            table_type="transient"
-                        )
-                        if not success:
-                            raise ValueError(
-                                f"Failed to write results to table {self._full_results_table_name}"
+                with tracer.start_as_current_span("apply_results"):
+                    for attempt in Retrying(stop=stop_after_attempt(30),wait=wait_fixed(2),reraise=True,retry=retry_if_exception_message(match=".*(is being|was) committed.*")):
+                        with attempt:
+                            logger.debug(
+                                f"Applying {len(results_df)} results to {self._full_results_table_name}"
                             )
-                        logger.debug(
-                            f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
-                        )
-                        # temp tables aren't allowed
-                        # snowflake_df = self._session.create_dataframe(results_df)
-                        # snowflake_df.write.save_as_table(table_name=temp_table,
-                        #                                mode='append',
-                        #                                column_order='index',
-                        #                                #create_temp_table=True
-                        #                                )
-                        for stream_name in stream_names:
-                            self._results_exist[stream_name] = True
+                            # try setting parquet engine here, since the engine parameter does not seem to make it through to the write_pandas function
+                            success, nchunks, nrows, _ = write_pandas(
+                                conn=self._session._conn._cursor.connection,  # pylint: disable=protected-access
+                                df=results_df,
+                                table_name=self._full_results_table_name,
+                                quote_identifiers=False,  # already done in get_temp_table_name
+                                # schema='INBOUND_RAW', # it seems to be ok to provide schema in the table name
+                                table_type="transient"
+                            )
+                            if not success:
+                                raise ValueError(
+                                    f"Failed to write results to table {self._full_results_table_name}"
+                                )
+                            logger.debug(
+                                f"Wrote {nrows} rows and {nchunks} chunks to table {self._full_results_table_name}"
+                            )
+                            # temp tables aren't allowed
+                            # snowflake_df = self._session.create_dataframe(results_df)
+                            # snowflake_df.write.save_as_table(table_name=temp_table,
+                            #                                mode='append',
+                            #                                column_order='index',
+                            #                                #create_temp_table=True
+                            #                                )
+                            for stream_name in stream_names:
+                                self._results_exist[stream_name] = True
         else:
             logger.debug("Results dataframe is empty, not applying")
 
@@ -2137,14 +2140,16 @@ def __managed_inbound_processing_worker(
             logger.debug(f"stream returned from queue: {stream}")
             # restore the first argument, was originally the dataframe/generator but now it's the appropriately sized dataframe
             try:
-                logger.debug(f"worker {worker_index} processing stream {stream.stream_name}, invoking plugin class method {method.__name__}")
-                result = method(plugin_class_obj, *(stream, *method_args), **method_kwargs)
-                logger.debug(f"worker {worker_index} completed processing stream {stream.stream_name}")
-                if result is not None and result is False:
-                    logger.info(f"worker {worker_index} requested that {stream.stream_name} be not marked as complete")
-                else:
-                    logger.info(f"worker {worker_index} marking stream {stream.stream_name} as complete")
-                    plugin_class_obj._sync_request.mark_stream_complete(stream.stream_name)
+                with tracer.start_as_current_span("managed_inbound_processing") as managed_inbound_processing_span:
+                    logger.debug(f"worker {worker_index} processing stream {stream.stream_name}, invoking plugin class method {method.__name__}")
+                    managed_inbound_processing_span.set_attribute("stream_name", stream.stream_name)
+                    result = method(plugin_class_obj, *(stream, *method_args), **method_kwargs)
+                    logger.debug(f"worker {worker_index} completed processing stream {stream.stream_name}")
+                    if result is not None and result is False:
+                        logger.info(f"worker {worker_index} requested that {stream.stream_name} be not marked as complete")
+                    else:
+                        logger.info(f"worker {worker_index} marking stream {stream.stream_name} as complete")
+                        plugin_class_obj._sync_request.mark_stream_complete(stream.stream_name)
             except InterruptedWhileWaitingException:
                 # If an inbound run is cancelled while waiting for rate limiting, this should mean that
                 # the cancellation is handled elsewhere, so we don't need to do anything special here other than stop waiting

{omnata_plugin_runtime-0.8.0a192 → omnata_plugin_runtime-0.8.0a194}/src/omnata_plugin_runtime/plugin_entrypoints.py

@@ -39,7 +39,7 @@ from .rate_limiting import ApiLimits, RateLimitState
 from opentelemetry import trace
 
 IMPORT_DIRECTORY_NAME = "snowflake_import_directory"
-tracer = trace.get_tracer(__name__)
+tracer = trace.get_tracer('omnata_plugin_runtime')
 
 class PluginEntrypoint:
     """