omnata-plugin-runtime 0.11.5a322__tar.gz → 0.11.6a323__tar.gz

{omnata_plugin_runtime-0.11.5a322 → omnata_plugin_runtime-0.11.6a323}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: omnata-plugin-runtime
-Version: 0.11.5a322
+Version: 0.11.6a323
 Summary: Classes and common runtime components for building and running Omnata Plugins
 License-File: LICENSE
 Author: James Weakley

{omnata_plugin_runtime-0.11.5a322 → omnata_plugin_runtime-0.11.6a323}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "omnata-plugin-runtime"
-version = "0.11.5-a322"
+version = "0.11.6-a323"
 description = "Classes and common runtime components for building and running Omnata Plugins"
 authors = ["James Weakley <james.weakley@omnata.com>"]
 readme = "README.md"

{omnata_plugin_runtime-0.11.5a322 → omnata_plugin_runtime-0.11.6a323}/src/omnata_plugin_runtime/logging.py

@@ -9,9 +9,10 @@ from typing import Dict, List, Optional
 from snowflake.snowpark import Session
 from pydantic import ValidationError
 from snowflake import telemetry
-from opentelemetry import trace
+from opentelemetry import trace, metrics
 
 tracer = trace.get_tracer('omnata_plugin_runtime')
+meter = metrics.get_meter('omnata_plugin_runtime')
 
 class CustomLoggerAdapter(logging.LoggerAdapter):
     """

{omnata_plugin_runtime-0.11.5a322 → omnata_plugin_runtime-0.11.6a323}/src/omnata_plugin_runtime/omnata_plugin.py

@@ -48,7 +48,7 @@ from snowflake.snowpark import Session
 from snowflake.snowpark.functions import col
 from tenacity import Retrying, stop_after_attempt, wait_fixed, retry_if_exception_message
 
-from .logging import OmnataPluginLogHandler, logger, tracer
+from .logging import OmnataPluginLogHandler, logger, tracer, meter
 from opentelemetry import context
 import math
 import numpy as np
@@ -2330,6 +2330,15 @@ def __managed_inbound_processing_worker(
         try:
             stream: StoredStreamConfiguration = streams_queue.get_nowait()
             logger.debug(f"stream returned from queue: {stream}")
+            sync_request: InboundSyncRequest = cast(
+                InboundSyncRequest, plugin_class_obj._sync_request
+            )  # pylint: disable=protected-access
+            stream_duration_counter = meter.create_histogram(
+                name="omnata.sync_run.stream_duration",
+                description="The duration of stream processing",
+                unit="s",
+            )
+            start_time = time.time()
             # restore the first argument, was originally the dataframe/generator but now it's the appropriately sized dataframe
             try:
                 with tracer.start_as_current_span("managed_inbound_processing") as managed_inbound_processing_span:
@@ -2341,7 +2350,7 @@ def __managed_inbound_processing_worker(
                         logger.info(f"worker {worker_index} requested that {stream.stream_name} be not marked as complete")
                     else:
                         logger.info(f"worker {worker_index} marking stream {stream.stream_name} as complete")
-                        plugin_class_obj._sync_request.mark_stream_complete(stream.stream_name)
+                        sync_request.mark_stream_complete(stream.stream_name)
             except InterruptedWhileWaitingException:
                 # If an inbound run is cancelled while waiting for rate limiting, this should mean that
                 # the cancellation is handled elsewhere, so we don't need to do anything special here other than stop waiting
@@ -2361,6 +2370,19 @@ def __managed_inbound_processing_worker(
                     omnata_plugin_logger.error(f"{type(e).__name__} syncing stream {stream.stream_name}", 
                                 exc_info=True, 
                                 extra={'stream_name':stream.stream_name})
+            finally:
+                duration = time.time() - start_time
+                stream_duration_counter.record(
+                    duration,
+                    attributes={
+                        "stream_name": stream.stream_name,
+                        "sync_run_id": str(sync_request._run_id),
+                        "sync_id": str(sync_request._sync_id),
+                        "branch_name": str(sync_request._branch_name) if sync_request._branch_name is not None else 'main',
+                        "sync_direction": "inbound",
+                        "plugin_id": plugin_class_obj.get_manifest().plugin_id,
+                    },
+                )
         except queue.Empty:
             logger.debug("streams queue is empty")
             return