openmodule 14.0.2__tar.gz → 14.0.4__tar.gz

{openmodule-14.0.2 → openmodule-14.0.4}/ChangeLog

@@ -1,6 +1,16 @@
 CHANGES
 =======
 
+v14.0.4
+-------
+
+* OM-945 Fix Trace parent in ZMQMessage
+
+v14.0.3
+-------
+
+* OM-941 Sentry Bugfix
+
 v14.0.2
 -------
 
@@ -164,9 +174,6 @@ v12.0.0.rc0
 * fixed testcases and removed backend (deprecated in v11)
 * newer settings-models
 * settings models update
-* made gate\_control/day\_mode have no scope
-* schedule exports utility
-* SettingsProvider docs
 
 v11.1.1
 -------
@@ -194,15 +201,8 @@ v11.0.2
 * Update database.md [skip ci]
 * Update database.md [skip ci]
 * Update database.md [skip ci]
-* Update database.md [skip ci]
 * Major Refactoring and many new functionality (v2 parking system refactor and redesign)
 
 v10.0.5
 -------
 
-* fixed problem in presence listener
-* README for settings + backend
-
-v10.0.4
--------
-

{openmodule-14.0.2/openmodule.egg-info → openmodule-14.0.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 14.0.2
+Version: 14.0.4
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

{openmodule-14.0.2 → openmodule-14.0.4}/docs/known_issues.md

@@ -5,7 +5,20 @@ version. The changelog may contain many other notes aswell.
 
 The version numbers below always note the _version through which the issue was fixed_.
 
+## 14.0.4
+
+* fixed ZMQMessage using `sentry-trace` and `baggage` from wrong sentry span
+
+## 14.0.3
+
+* fixed not traced topics (io) using sentry scope from different thread leading to possible race condition
+
+## 14.0.2
+
+* fixed sentry trace for multithreaded `MessageDispatcher` and `RPCServer`
+
 ## 13.6.0
+
 * fixed message dispatcher trying to handle messages after shutdown was called
 
 ## 13.1.2

{openmodule-14.0.2 → openmodule-14.0.4}/openmodule/models/base.py

@@ -139,14 +139,16 @@ class ZMQMessage(OpenModuleModel):
     timestamp: datetime = Field(default_factory=lambda: datetime.utcnow())
     name: str = Field(default_factory=lambda: settings.NAME)
     type: str
-    baggage: Optional[str] = Field(default_factory=sentry.get_baggage)
-    sentry_trace: Optional[str] = Field(default_factory=sentry.get_traceparent, alias="sentry-trace")
+    baggage: Optional[str] = None
+    sentry_trace: Optional[str] = Field(default=None, alias="sentry-trace")
 
     _tz_timestamp = timezone_validator("timestamp")
 
     def publish_on_topic(self, pub_socket: zmq.Socket, topic: str):
         assert isinstance(topic, str), "topic must be a string"
 
+        self.baggage = sentry.get_baggage()
+        self.sentry_trace = sentry.get_traceparent()
         pub_socket.send_multipart((topic.encode("utf8"), self.json_bytes()))
 
     def dict(self, **kwargs):

{openmodule-14.0.2 → openmodule-14.0.4}/openmodule/sentry.py

@@ -36,8 +36,6 @@ def continue_trace_zmq_message_receive(topic: str, message: dict, **kwargs):
     :param topic: The topic of the message (as received from the ZMQ socket and decoded)
     :param message: The ZMQMessage as a dict (as received from the ZMQ socket and json loaded)
     """
-    if topic in _topics_to_ignore:
-        return noop_context()
     return _continue_trace_zmq_message(topic, message, "topic.receive", **kwargs)
 
 
@@ -59,6 +57,7 @@ def trace(
         op: Union[sentry_sdk.consts.OP, str] = sentry_sdk.consts.OP.FUNCTION,
         trace_headers: Optional[dict] = None,
         force_new_transaction: bool = False,
+        force_no_trace: bool = False,
         **span_kwargs) -> Callable[[Callable], Callable]:
     ...
 
@@ -69,6 +68,7 @@ def trace(
         op: Union[sentry_sdk.consts.OP, str] = sentry_sdk.consts.OP.FUNCTION,
         trace_headers: Optional[dict] = None,
         force_new_transaction: bool = False,
+        force_no_trace: bool = False,
         **span_kwargs) -> Union[Callable, Generator[sentry_sdk.tracing.Span, None, None]]:
     ...
 
@@ -79,6 +79,7 @@ def trace(
         op: Union[sentry_sdk.consts.OP, str] = sentry_sdk.consts.OP.FUNCTION,
         trace_headers: Optional[dict] = None,
         force_new_transaction: bool = False,
+        force_no_trace: bool = False,
         **span_kwargs) -> Callable:
     ...
 
@@ -88,6 +89,7 @@ def trace(
         op: Union[sentry_sdk.consts.OP, str] = sentry_sdk.consts.OP.FUNCTION,
         trace_headers: Optional[dict] = None,
         force_new_transaction: bool = False,
+        force_no_trace: bool = False,
         **span_kwargs):
     """
     Trace a function or a block of code.
@@ -97,6 +99,7 @@ def trace(
     span_kwargs["op"] = op
     span_kwargs["trace_headers"] = trace_headers
     span_kwargs["force_new_transaction"] = force_new_transaction
+    span_kwargs["force_no_trace"] = force_no_trace
     if name is None:
         return _Trace(**span_kwargs)
     elif callable(name):
@@ -125,7 +128,7 @@ def get_trace_headers(empty_if_not_sampled: bool = False) -> dict:
         span = sentry_sdk.get_current_span()
         if span and not span.sampled:
             return {}
-    return {"sentry-trace": sentry_sdk.get_traceparent(), "baggage": sentry_sdk.get_baggage()}
+    return {"sentry-trace": get_traceparent(), "baggage": get_baggage()}
 
 
 class _Trace:
@@ -137,9 +140,11 @@ class _Trace:
     force_new_transaction: bool
     ctx: Any
 
-    def __init__(self, *, trace_headers: Optional[dict] = None, force_new_transaction: bool = False, **span_kwargs):
+    def __init__(self, *, trace_headers: Optional[dict] = None, force_new_transaction: bool = False,
+                 force_no_trace: bool = False, **span_kwargs):
         self.trace_headers = trace_headers
         self.force_new_transaction = force_new_transaction
+        self.force_no_trace = force_no_trace
         self.span_kwargs = span_kwargs
 
     @contextmanager
@@ -149,9 +154,7 @@ class _Trace:
         Otherwise, start a new span.
         If given a trace_headers dict, it will continue the trace with the given headers.
         """
-        if self.span_kwargs.get("name") in _functions_to_ignore:
-            yield
-            return
+        force_no_trace = self.force_no_trace or self.span_kwargs.get("name") in _functions_to_ignore
 
         def set_context():
             for key, value in extras.items():
@@ -166,15 +169,19 @@ class _Trace:
         span = sentry_sdk.get_current_span()
         if span is None or self.force_new_transaction:
             with sentry_sdk.isolation_scope():
-                transaction: Optional[sentry_sdk.tracing.Transaction] = None
-                if self.trace_headers and "sentry-trace" in self.trace_headers:
-                    transaction = sentry_sdk.continue_trace(self.trace_headers, **self.span_kwargs)
-                with sentry_sdk.start_transaction(transaction=transaction, **self.span_kwargs) as transaction:
-                    if transaction:
-                        transaction.description = transaction.name
+                if force_no_trace:
                     set_context()
                     yield
-        elif span.sampled:
+                else:
+                    transaction: Optional[sentry_sdk.tracing.Transaction] = None
+                    if self.trace_headers and "sentry-trace" in self.trace_headers:
+                        transaction = sentry_sdk.continue_trace(self.trace_headers, **self.span_kwargs)
+                    with sentry_sdk.start_transaction(transaction=transaction, **self.span_kwargs) as transaction:
+                        if transaction:
+                            transaction.description = transaction.name
+                        set_context()
+                        yield
+        elif not force_no_trace and span.sampled:
             self.span_kwargs.pop("source", None)
             with sentry_sdk.start_span(**self.span_kwargs):
                 set_context()
@@ -217,7 +224,8 @@ class _Trace:
 
 def _continue_trace_zmq_message(topic: str, message: dict, op: str, **kwargs):
     name = f"{topic}/{message.get('type', '')}"
-    return _Trace(trace_headers=message, op=op, name=name, source="zmq", origin="auto.zmq", **kwargs)
+    return _Trace(trace_headers=message, op=op, name=name, source="zmq", origin="auto.zmq",
+                  force_no_trace=topic in _topics_to_ignore, **kwargs)
 
 
 class StoringTransport(sentry_sdk.transport.HttpTransport):

{openmodule-14.0.2 → openmodule-14.0.4}/openmodule/utils/kv_store.py

@@ -3,6 +3,7 @@ import logging
 import random
 import threading
 import time
+import uuid
 import warnings
 from typing import Type, Optional, List, Union, Dict
 
@@ -193,7 +194,7 @@ class KVStore:
         additional_kvs = current_kvs
         return KVSyncResponse(additions=additional_kvs, changes=changed_kvs, missing=missing_kvs)
 
-    def _log_sync_rpc_response(self, rpc_entry: RPCClient.RPCEntry) -> bool:
+    def _log_sync_rpc_response(self, rpc_entry: RPCClient.RPCEntry, conn_id: str) -> bool:
         """
         Waits for the RPC to finish and logs the response
         :return: if successful
@@ -203,11 +204,22 @@ class KVStore:
             rpc_entry.result(ServerSyncResponse)
             return True
         except RPCClient.TimeoutError:
-            self.log.error("Timeout syncing kvs")
+            if core().connection_listener.changed(conn_id):
+                self.log.warning("Connection loss during sync")
+            else:
+                self.log.error("Timeout syncing kvs")
         except RPCClient.CancelError:
-            self.log.error("Cancelled waiting for result of syncing kvs")
+            if core().connection_listener.changed(conn_id):
+                self.log.warning("Connection loss during sync")
+            else:
+                self.log.error("Cancelled waiting for result of syncing kvs")
         except RPCClient.ValidationError:
             self.log.exception("Error parsing sync response")
+        except RPCClient.RPCServerError as e:
+            if len(e.args) > 0 and e.args[0] == "request_timeout" and core().connection_listener.changed(conn_id):
+                self.log.warning("Connection loss during sync")
+            else:
+                self.log.exception("Error on server side in sync")
         except (RPCClient.ServerHandlerError, RPCClient.ServerValidationError, RPCClient.ServerFilterError):
             self.log.exception("Error on server side in sync")
         except Exception:
@@ -215,9 +227,9 @@ class KVStore:
         return False
 
     @sentry.trace
-    def sync(self) -> bool:
+    def sync(self, conn_id: str) -> bool:
         self.sync_rpc_entry = self.sync_with_server()
-        success = self._log_sync_rpc_response(self.sync_rpc_entry)
+        success = self._log_sync_rpc_response(self.sync_rpc_entry, conn_id)
         self.sync_rpc_entry = None
         if success:  # if sync was successful cancel retries
             return True
@@ -291,6 +303,7 @@ class KVStoreHandler:
 
     def run(self):
         """We only sync on startup and on reconnect"""
+        conn_id = uuid.uuid4().hex
         try:
             while self.running:
                 if self._wait_for_change_to_online():
@@ -305,7 +318,8 @@ class KVStoreHandler:
                                                                         ConnectionStatus.online):
                                 # cancel trying to sync if offline
                                 break
-                            if store.sync():
+                            core().connection_listener.changed(conn_id)
+                            if store.sync(conn_id):
                                 # if sync was successful cancel retries
                                 break
                             # we try again as long we are still alive
@@ -347,7 +361,7 @@ class KVStoreWithChangedNotification(KVStore):
         changed += list(old_values.keys())
         return changed
 
-    def _send_changed_notification(self, new_values: Dict[str, str], old_values: Dict[str, str]): # pragma: no cover
+    def _send_changed_notification(self, new_values: Dict[str, str], old_values: Dict[str, str]):  # pragma: no cover
         """
         consider using self._find_changed_kvs when implementing this
         also make sure your db model has a meaningful `comparison_value` method

{openmodule-14.0.2 → openmodule-14.0.4/openmodule.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: openmodule
-Version: 14.0.2
+Version: 14.0.4
 Summary: Libraries for developing the arivo openmodule
 Home-page: https://gitlab.com/arivo-public/device-python/openmodule.git
 Author: ARIVO

openmodule-14.0.4/openmodule.egg-info/pbr.json

@@ -0,0 +1 @@
+{"git_version": "3d9ddfd", "is_release": true}

{openmodule-14.0.2 → openmodule-14.0.4}/tests/test_model.py

@@ -137,8 +137,7 @@ class ZMQMessageTestCase(TestCase):
         self.assertEqual(res3, res4)
 
         res1.pop("timestamp")
-        sentry_trace = sentry.get_traceparent()
-        self.assertDictEqual(res1, {"name": "tä\nst", "type": "tö\tst", "sentry-trace": sentry_trace})
+        self.assertDictEqual(res1, {"name": "tä\nst", "type": "tö\tst"})
 
 
 class CostEntryDataTextCase(TestCase):

{openmodule-14.0.2 → openmodule-14.0.4}/tests/test_sentry.py

@@ -173,7 +173,6 @@ class SentryInitTestCase(ZMQTestMixin):
         self.assertEqual(sentry._topics_to_ignore, ["io"])
         self.assertEqual(sentry._functions_to_ignore, ["message_handler.openmodule.utils.io.IoListener._on_io_message"])
 
-
     @override_settings(NAME="test")
     def test_service_name_generation(self):
         settings.NAME = "om_service_test_1"
@@ -449,6 +448,7 @@ class SentryTestCase(SentryTestMixin):
         self.assertEqual(len(envelopes[0].items), 1)
         self.assertEqual(envelopes[0].items[0].payload.json["transaction"], "not-test-ignore")
 
+
 class SentryInDispatcherTestCase(SentryTestMixin):
     def setUp(self):
         super().setUp()
@@ -488,6 +488,15 @@ class SentryInDispatcherTestCase(SentryTestMixin):
         with self.assertRaises(TimeoutError):
             self.get_sent_envelopes()
 
+    def test_io_topic_creates_scope(self):
+        scope = sentry_sdk.get_isolation_scope()
+        with sentry.continue_trace_zmq_message_receive(
+                "io", {"sentry-trace": "e6b4de069f9a4e03b7012bb1eec1af08-a7d7309ad4ccc9ec-1"}):
+            self.assertNotEqual(scope, sentry_sdk.get_isolation_scope())
+        self.assertEqual(scope, sentry_sdk.get_isolation_scope())
+        with self.assertRaises(TimeoutError):
+            self.get_sent_envelopes()
+
     def test_multi_worker_dispatcher(self):
         def handler(*_):
             envelopes_got.wait(1)
@@ -524,15 +533,27 @@ class SentryUtilsTestCase(SentryTestMixin):
         sentry.init_sentry("http://test@test/1")
 
     def test_zmq_message_fields(self):
+        zmq_socket = mock.MagicMock()
+        message = ZMQMessage(type="test")
+        self.assertIsNone(message.sentry_trace)
+        self.assertIsNone(message.baggage)
         with sentry.trace("test", sampled=True):
-            msg = ZMQMessage(type="test")
-        json_data = msg.json_bytes()
-        self.assertIn(b'"sentry-trace":', json_data)
-        self.assertIn(b'"baggage":', json_data)
+            message.publish_on_topic(zmq_socket, "test")
+            traceparent = sentry.get_traceparent()
+            baggage = sentry.get_baggage()
+            self.assertEqual(message.sentry_trace, traceparent)
+            self.assertEqual(message.baggage, baggage)
+            self.assertIsNotNone(message.sentry_trace)
+            self.assertIsNotNone(message.baggage)
+        sent_message = json.loads(zmq_socket.send_multipart.call_args[0][0][1])
+        self.assertEqual(sent_message["sentry-trace"], traceparent)
+        self.assertEqual(sent_message["baggage"], baggage)
 
     def test_continue_from_zmq_message(self):
+        zmq_socket = mock.MagicMock()
         with sentry.trace("test", sampled=True):
             msg = ZMQMessage(type="test_type")
+            msg.publish_on_topic(zmq_socket, "topic_test")
             trace_id = sentry_sdk.get_current_span().trace_id
         msg_data = json.loads(msg.json_bytes())
         with sentry.continue_trace_zmq_message_process("topic_test", msg_data):
@@ -759,7 +780,7 @@ class SentryTraceCpuUsageTestCase(TestCase):
         import psutil
         helper = SentryTraceHelper(self.execution_count, self.sleep_time)
 
-        self.processes.append(multiprocessing.Process(target=helper.sentry))
+        self.processes.append(multiprocessing.Process(target=helper.sentry_io))
         self.processes.append(multiprocessing.Process(target=helper.fake))
         for process in self.processes:
             process.start()
@@ -809,21 +830,45 @@ class SentryTraceHelper:
         sentry_sdk.start_transaction = self.fake_trace
         self.sentry()
 
+    @staticmethod
+    def make_core(traces_sample_rate=0.001):
+        settings.LOG_LEVEL = "WARNING"
+        settings.TESTING = True
+        settings.DEBUG = False
+        sentry_sdk.utils.logger.setLevel(logging.WARNING)
+        logging.basicConfig(level=logging.WARNING)
+        return init_openmodule(settings, wait_for_broker=False, traces_sample_rate=traces_sample_rate,
+                               sentry=True, dsn="http://test@test/1")
+
     def trace_all(self):
         self.sentry(traces_sample_rate=1.0)
 
+    @staticmethod
+    def _sentry_traceparent(traces_sample_rate=0.001):
+        return f"e6b4de069f9a4e03b7012bb1eec1af08-a7d7309ad4ccc9ec-{int(random.random() < traces_sample_rate)}"
+
+    def sentry_io(self, traces_sample_rate=0.001):
+        core = None
+        try:
+            core = self.make_core(traces_sample_rate)
+            IoListener(core.messages)
+            for _ in range(self.execution_count):
+                message = {"edge": 1, "gateway": {"gate": "test", "direction": "in"}, "type": "io", "pin": "1",
+                           "value": 1, "sentry-trace": self._sentry_traceparent(traces_sample_rate)}
+                with sentry.continue_trace_zmq_message_receive("io", message):
+                    core.messages.dispatch("io", message)
+                time.sleep(self.sleep_time)
+        finally:
+            if core:
+                shutdown_openmodule()
+
     def sentry(self, traces_sample_rate=0.001):
+        from openmodule.models.base import CostEntryData
+
         rpc_server = None
         core = None
         try:
-            from openmodule.models.base import CostEntryData
-            settings.LOG_LEVEL = "WARNING"
-            settings.TESTING = True
-            settings.DEBUG = False
-            sentry_sdk.utils.logger.setLevel(logging.WARNING)
-            logging.basicConfig(level=logging.WARNING)
-            core = init_openmodule(settings, wait_for_broker=False, traces_sample_rate=traces_sample_rate,
-                                   sentry=True, dsn="http://test@test/1")
+            core = self.make_core(traces_sample_rate)
             sentry_sdk.get_client().transport.envelope_store = "/tmp/envelope_store"
             rpc_server = RPCServer(core.context)
             rpc_server.register_handler("test", "test", CostEntryData, CostEntryData, self.fake_handler,
@@ -831,8 +876,7 @@ class SentryTraceHelper:
             for _ in range(self.execution_count):
                 # noinspection PyProtectedMember
                 rpc_server._process_rpc_message("rpc-req-test", {
-                    "sentry-trace": f"e6b4de069f9a4e03b7012bb1eec1af08-a7d7309ad4ccc9ec-"
-                                    f"{int(random.random() < traces_sample_rate)}",
+                    "sentry-trace": self._sentry_traceparent(traces_sample_rate),
                     "rpc_id": "ef9a5e77-bc95-404c-94de-9cf7b326c4d5", "resource": settings.RESOURCE, "type": "test",
                     "request": {"entry_type": "rate_change", "value": "test1", "group": "test", "account_id": "test"}})
                 time.sleep(self.sleep_time)

{openmodule-14.0.2 → openmodule-14.0.4}/tests/test_utils_kv_store.py

@@ -1,5 +1,6 @@
 import threading
 import time
+import uuid
 from typing import List, Dict
 
 from sqlalchemy import Column, String
@@ -7,7 +8,7 @@ from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.declarative import declarative_base
 
 from openmodule.config import settings
-from openmodule.connection_status import ConnectionStatus
+from openmodule.connection_status import ConnectionStatus, ConnectionStatusMessage
 from openmodule.core import core
 from openmodule.models.kv_store import KVSetRequest, KVSetRequestKV, KVSyncRequest, KVSetResponse, KVSyncResponse, \
     ServerSyncResponse
@@ -317,8 +318,10 @@ class KVStoreRPCTestCase(SQLiteTestMixin, RPCServerTestMixin, OpenModuleCoreTest
             session.commit()
 
         self.kv_store.rpc_client = MockRPCClient(responses={("rpc-websocket", "server_rpc"): start_sync()})
+        conn_id = uuid.uuid4().hex
+        self.core.connection_listener.changed(conn_id)
         entry = self.kv_store.sync_with_server()
-        self.kv_store._log_sync_rpc_response(entry)
+        self.kv_store._log_sync_rpc_response(entry, conn_id)
 
     def test_sync_errors(self):
         def raise_error(*_):
@@ -327,31 +330,51 @@ class KVStoreRPCTestCase(SQLiteTestMixin, RPCServerTestMixin, OpenModuleCoreTest
         def wrong_response(*_):
             return {}
 
+        conn_id = uuid.uuid4().hex
+
         # status error
+        self.core.connection_listener.changed(conn_id)
         self.rpc_server.register_handler("rpc-websocket", "server_rpc", ServerRPCRequest, ServerSyncResponse,
                                          raise_error, register_schema=False)
         entry = self.kv_store.sync_with_server()
         with self.assertLogs() as cm:
-            success = self.kv_store._log_sync_rpc_response(entry)
+            success = self.kv_store._log_sync_rpc_response(entry, conn_id)
         self.assertIn("Error on server side in sync", str(cm.output))
         self.assertFalse(success)
 
         # Un-parseable response
+        self.core.connection_listener.changed(conn_id)
         entry = RPCClient.RPCEntry(0)
         entry.response = "test"
         with self.assertLogs() as cm:
-            success = self.kv_store._log_sync_rpc_response(entry)
+            success = self.kv_store._log_sync_rpc_response(entry, conn_id)
         self.assertIn("Unexpected exception in sync response", str(cm.output))
         self.assertFalse(success)
 
         # timeout
+        self.core.connection_listener.changed(conn_id)
         del self.rpc_server.handlers[("rpc-websocket", "server_rpc")]
         entry = self.kv_store.sync_with_server()
         with self.assertLogs() as cm:
-            success = self.kv_store._log_sync_rpc_response(entry)
+            success = self.kv_store._log_sync_rpc_response(entry, conn_id)
         self.assertIn("Timeout syncing kvs", str(cm.output))
         self.assertFalse(success)
 
+    def test_connection_loss_warning(self):
+        conn_id = uuid.uuid4().hex
+        self.core.connection_listener._process_connection_status(
+            ConnectionStatusMessage(connected=ConnectionStatus.online))
+        self.core.connection_listener.changed(conn_id)
+        self.core.connection_listener._process_connection_status(
+            ConnectionStatusMessage(connected=ConnectionStatus.shutdown))
+        entry = self.kv_store.sync_with_server()
+        with self.assertLogs() as cm:
+            success = self.kv_store._log_sync_rpc_response(entry, conn_id)
+        self.assertIn("Connection loss during sync", str(cm.output))
+        self.assertEqual(len(cm.output), 1)
+        self.assertFalse(success)
+
+
 
 class KVStoreSyncTriggerTestCase(OpenModuleCoreTestMixin):
     def setUp(self) -> None:

openmodule-14.0.2/openmodule.egg-info/pbr.json

@@ -1 +0,0 @@
-{"git_version": "3399243", "is_release": true}