switch-api 0.6.17__tar.gz → 0.6.18__tar.gz

{switch_api-0.6.17 → switch_api-0.6.18}/HISTORY.md

@@ -1,5 +1,18 @@
 # History
 
+#0.6.18
+### Changed
+In the `realtime` module:
+- Updated `submit_realtime_updates()` to use the passed `timeout` value when waiting for a retained MQTT message.
+- Added Redis-backed optimistic concurrency support for realtime dataset updates using WATCH/MULTI/EXEC.
+- Added `set_realtime_redis_config()` to configure Redis as the authoritative state store before publishing updates to MQTT.
+- Added `redis` package dependency for Redis-backed realtime state updates.
+- Added a maximum of 3 retained-message subscription attempts for connection or subscribe failures in the MQTT fallback path.
+- Preserved new-topic behavior: when no retained message exists and Redis is not configured, the provided dataframe is used and published as the initial retained dataset.
+- Updated `subscribe_with_retained_sync()` to apply the passed timeout to the MQTT connection, check subscribe failures, and log when no retained topics are received within the wait period.
+- Added Redis timeout retry handling with a default maximum of 3 attempts for Redis realtime update operations.
+
+
 #0.6.17
 ### Changed
 In the `integration` module:

{switch_api-0.6.17 → switch_api-0.6.18}/PKG-INFO

@@ -1,11 +1,9 @@
-Metadata-Version: 2.1
+Metadata-Version: 2.4
 Name: switch_api
-Version: 0.6.17
+Version: 0.6.18
 Summary: A complete package for data ingestion into the Switch Automation Platform.
-Home-page: UNKNOWN
 Author: Switch Automation Pty Ltd.
 License: MIT License
-Platform: UNKNOWN
 Classifier: Development Status :: 2 - Pre-Alpha
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Intended Audience :: Other Audience
@@ -17,6 +15,28 @@ Requires-Python: >=3.8.0
 Description-Content-Type: text/markdown
 License-File: LICENCE
 License-File: AUTHORS.rst
+Requires-Dist: pandas==1.5.3
+Requires-Dist: requests
+Requires-Dist: azure-storage-blob
+Requires-Dist: pandera[io]==0.7.1
+Requires-Dist: azure-servicebus
+Requires-Dist: msal>=1.11.0
+Requires-Dist: paho-mqtt==1.6.1
+Requires-Dist: uvicorn==0.22.0
+Requires-Dist: fastapi==0.98.0
+Requires-Dist: pyodbc==4.0.39
+Requires-Dist: cachetools==5.5.0
+Requires-Dist: numexpr==2.10.2
+Requires-Dist: redis<6,>=4.5.5
+Dynamic: author
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
 
 # Switch Automation library for Python
 This is a package for data ingestion into the Switch Automation software platform. 
@@ -38,6 +58,19 @@ pip install switch_api
 
 # History
 
+#0.6.18
+### Changed
+In the `realtime` module:
+- Updated `submit_realtime_updates()` to use the passed `timeout` value when waiting for a retained MQTT message.
+- Added Redis-backed optimistic concurrency support for realtime dataset updates using WATCH/MULTI/EXEC.
+- Added `set_realtime_redis_config()` to configure Redis as the authoritative state store before publishing updates to MQTT.
+- Added `redis` package dependency for Redis-backed realtime state updates.
+- Added a maximum of 3 retained-message subscription attempts for connection or subscribe failures in the MQTT fallback path.
+- Preserved new-topic behavior: when no retained message exists and Redis is not configured, the provided dataframe is used and published as the initial retained dataset.
+- Updated `subscribe_with_retained_sync()` to apply the passed timeout to the MQTT connection, check subscribe failures, and log when no retained topics are received within the wait period.
+- Added Redis timeout retry handling with a default maximum of 3 attempts for Redis realtime update operations.
+
+
 #0.6.17
 ### Changed
 In the `integration` module:
@@ -1196,5 +1229,3 @@ Removed the `name_as_filename` and `treat_as_timeseries` parameter from the foll
 - `switch.integration.replace_data()`
 - `switch.integration.append_data()`
 - `switch.integration.upload_data()`
-
-

switch_api-0.6.18/pyproject.toml

@@ -0,0 +1,6 @@
+[build-system]
+requires = [
+    "setuptools>=64",
+    "wheel"
+]
+build-backend = "setuptools.build_meta"

{switch_api-0.6.17 → switch_api-0.6.18}/setup.py

@@ -16,11 +16,11 @@ setup(
     long_description_content_type='text/markdown',
     license='MIT License',
     name="switch_api",
-    version="0.6.17",
+    version="0.6.18",
     packages=find_packages(include=["switch_api", "switch_api.*"]),
     install_requires=['pandas==1.5.3', 'requests', 'azure-storage-blob', 'pandera[io]==0.7.1', 'azure-servicebus',
                       'msal>=1.11.0', 'paho-mqtt==1.6.1', 'uvicorn==0.22.0', 'fastapi==0.98.0', 'pyodbc==4.0.39', 'cachetools==5.5.0', 
-                      'numexpr==2.10.2'],
+                      'numexpr==2.10.2', 'redis>=4.5.5,<6'],
     python_requires=">=3.8.0",
     classifiers=[
         'Development Status :: 2 - Pre-Alpha',

{switch_api-0.6.17 → switch_api-0.6.18}/switch_api/__init__.py

@@ -10,7 +10,8 @@ Data Ingestion into the Switch Automation
 Complete package for ingestion data into the Switch Automation Platform.
 """
 __all__ = ['dataset', 'analytics', 'pipeline', 'error_handlers', 'integration',
-           'initialize', 'cache', 'extensions', 'email', 'controls', 'realtime', '_compute', 'generate_filepath', 'set_mqtt_broker_config']
+           'initialize', 'cache', 'extensions', 'email', 'controls', 'realtime', '_compute', 'generate_filepath',
+           'set_mqtt_broker_config', 'set_realtime_redis_config']
 
 from .initialize import initialize
 from . import dataset
@@ -27,6 +28,7 @@ from . import realtime
 from . import _compute
 from ._utils._utils import generate_filepath
 from ._utils._mqtt import set_mqtt_broker_config
+from .realtime import set_realtime_redis_config
 from . import _guide
 
 
@@ -42,4 +44,4 @@ from . import _guide
 # log.addHandler(_ch)
 # https://dev.to/joaomcteixeira/setting-up-python-logging-for-a-library-app-6ml
 
-__version__ = "0.6.17"
+__version__ = "0.6.18"

{switch_api-0.6.17 → switch_api-0.6.18}/switch_api/realtime/__init__.py

@@ -8,5 +8,6 @@ A module for sending control request of sensors.
 """
 
 from .realtime import submit_realtime_updates
+from ._redis import set_realtime_redis_config
 
-__all__ = ['submit_realtime_updates']
+__all__ = ['submit_realtime_updates', 'set_realtime_redis_config']

{switch_api-0.6.17 → switch_api-0.6.18}/switch_api/realtime/_mqtt.py

@@ -7,7 +7,6 @@
 A module containing the SwithRealtime class and associated methods referenced by the controls module in the package. This
 module is not directly referenced by end users.
 """
-import json
 import logging
 import sys
 import threading
@@ -109,8 +108,6 @@ class SwitchRealtime:
         if topics is None:
             raise ValueError("topics cannot be None")
 
-        self.connect()
-
         topics = list(topics)
 
         if not topics:
@@ -122,6 +119,11 @@ class SwitchRealtime:
         if not isinstance(timeout, int) or timeout <= 0:
             raise ValueError("timeout must be a positive integer")
 
+        if not self.connect(timeout=timeout):
+            raise TimeoutError(
+                f"Could not connect to MQTT Broker within {timeout} seconds."
+            )
+
         # --- Initialize tracking ---
         self._expected_topics = set(topics)
         self._received_retained = set()
@@ -129,19 +131,32 @@ class SwitchRealtime:
 
         # --- Subscribe ---
         for topic in topics:
-            self.client.subscribe(topic)
+            result, _ = self.client.subscribe(topic)
+            if result != mqtt.MQTT_ERR_SUCCESS:
+                raise RuntimeError(
+                    f"Failed to subscribe to {topic}. MQTT result code: {result}"
+                )
             logger.info(f"Subscribed to = {topic}")
 
         # --- Wait until all retained received OR timeout ---
-        self._retained_event.wait(timeout=timeout)
+        received_all = self._retained_event.wait(timeout=timeout)
 
         # --- Return only retained dataset ---
-        return {
+        retained_dataset = {
             topic: self.topic_state[topic]["payload"]
             for topic in self._received_retained
             if topic in self.topic_state
         }
 
+        if not received_all:
+            missing_topics = self._expected_topics - self._received_retained
+            logger.info(
+                f"No retained messages received within {timeout} seconds "
+                f"for topics: {sorted(missing_topics)}"
+            )
+
+        return retained_dataset
+
 
     def publish(self, topic: str, payload: str, qos: int = 1, retain: bool = True):
         """
@@ -151,7 +166,11 @@ class SwitchRealtime:
         result = self.client.publish(topic, payload, qos=qos, retain=retain)
 
         if result.rc != mqtt.MQTT_ERR_SUCCESS:
-            logger.error(f"Failed to publish to {topic} : {json.loads(payload)}")
+            message = f"Failed to publish to {topic}. MQTT result code: {result.rc}"
+            logger.error(message)
+            raise RuntimeError(message)
+
+        return result
 
 
 
@@ -202,4 +221,4 @@ class SwitchRealtime:
                 self._retained_event.set()
 
         else:
-           logger.info(f'MQTT Message - {topic} : {payload}')
+           logger.info(f'MQTT Message - {topic} : {payload}')

switch_api-0.6.18/switch_api/realtime/_redis.py

@@ -0,0 +1,429 @@
+# -------------------------------------------------------------------------
+# Copyright (c) Switch Automation Pty Ltd. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+"""
+Redis state helpers for concurrency-safe realtime dataset updates.
+"""
+import logging
+import random
+import sys
+import time
+from typing import Callable
+
+# -----------------------------------------------------------------------------
+# Logging Setup
+# -----------------------------------------------------------------------------
+logger = logging.getLogger(__name__)
+logger.setLevel(logging.DEBUG)
+
+if not logger.handlers:
+    console_handler = logging.StreamHandler(stream=sys.stdout)
+    console_handler.setLevel(logging.INFO)
+
+    formatter = logging.Formatter(
+        "%(asctime)s  %(name)s.%(funcName)s  %(levelname)s: %(message)s",
+        datefmt="%Y-%m-%dT%H:%M:%S",
+    )
+    console_handler.setFormatter(formatter)
+    logger.addHandler(console_handler)
+
+
+global _redis_config
+global _redis_client
+
+_redis_config = {
+    "host": "",
+    "port": 6380,
+    "username": None,
+    "password": "",
+    "ssl": True,
+    "db": 0,
+    "key_prefix": "mqtt-state",
+    "max_retries": 10,
+    "timeout_max_retries": 3,
+    "retry_min_delay": 0.01,
+    "retry_max_delay": 0.05,
+    "socket_timeout": None,
+    "socket_connect_timeout": None,
+}
+_redis_client = None
+
+
+def _log_json_value(label: str, value):
+    if value is None:
+        logger.info(f"{label}: <None>")
+    else:
+        logger.info(f"{label}: {value}")
+
+
+def _get_redis_modules():
+    try:
+        import redis
+        from redis.exceptions import TimeoutError as RedisTimeoutError
+        from redis.exceptions import WatchError
+    except ImportError as e:
+        raise ImportError(
+            "Redis realtime state support requires the 'redis' package."
+        ) from e
+
+    return redis, WatchError, RedisTimeoutError
+
+
+def _is_redis_timeout_error(error: Exception):
+    _, _, RedisTimeoutError = _get_redis_modules()
+    return isinstance(error, (RedisTimeoutError, TimeoutError))
+
+
+def _reset_redis_client():
+    global _redis_client
+
+    if _redis_client is not None:
+        try:
+            _redis_client.close()
+        except Exception as e:
+            logger.warning(f"Error closing Redis realtime client after timeout: {e}")
+
+    _redis_client = None
+
+
+def set_realtime_redis_config(
+    host: str,
+    password: str = "",
+    port: int = 6380,
+    username: str = None,
+    ssl: bool = True,
+    db: int = 0,
+    key_prefix: str = "mqtt-state",
+    max_retries: int = 10,
+    timeout_max_retries: int = 3,
+    retry_min_delay: float = 0.01,
+    retry_max_delay: float = 0.05,
+    socket_timeout: int = None,
+    socket_connect_timeout: int = None,
+):
+    """
+    Configure Redis state storage for realtime MQTT dataset updates.
+
+    This method enables Redis-backed concurrency protection for
+    `submit_realtime_updates()`.
+
+    When Redis is configured, `submit_realtime_updates()` treats Redis as the
+    authoritative dataset store. Dataset updates are applied using Redis
+    optimistic concurrency with WATCH/MULTI/EXEC, then the final serialized
+    dataset is published back to MQTT.
+
+    When Redis is not configured, `submit_realtime_updates()` falls back to
+    reading the retained MQTT message before publishing the updated dataset.
+
+    Redis Key Format
+    ----------------
+    A single Redis key is constructed in the format:
+
+        {key_prefix}:{topic}
+
+    The default key prefix is:
+
+        mqtt-state
+
+    Example:
+        If key_prefix = "mqtt-state"
+        and topic = "Realtime/123/temperature"
+
+        The resulting Redis key will be:
+            mqtt-state:Realtime/123/temperature
+
+    Parameters
+    ----------
+    host : str
+        Redis host name.
+        For Azure Redis, this is typically the cache host name, such as:
+            your-cache.redis.cache.windows.net
+        Must be a non-empty string.
+
+    password : str, optional
+        Redis access key or password.
+        Defaults to an empty string.
+
+    port : int, optional
+        Redis port.
+        Defaults to 6380, which is the SSL port commonly used by Azure Redis.
+        Must be greater than 0.
+
+    username : str, optional
+        Redis username.
+        Defaults to None.
+
+    ssl : bool, optional
+        Indicates whether SSL should be used for the Redis connection.
+        Defaults to True.
+
+    db : int, optional
+        Redis database number.
+        Defaults to 0.
+        Must be zero or a positive integer.
+
+    key_prefix : str, optional
+        Prefix used when constructing Redis dataset keys.
+        Defaults to "mqtt-state".
+        Must be a non-empty string.
+
+    max_retries : int, optional
+        Maximum number of optimistic concurrency retry attempts when Redis
+        detects that another service updated the dataset during the current
+        update attempt.
+        Defaults to 10.
+        Must be greater than 0.
+
+    timeout_max_retries : int, optional
+        Maximum number of Redis timeout retry attempts when a Redis operation
+        exceeds the configured socket timeout.
+        Defaults to 3.
+        Must be greater than 0.
+
+    retry_min_delay : float, optional
+        Minimum randomized delay, in seconds, before retrying after a Redis
+        WATCH conflict or Redis timeout.
+        Defaults to 0.01.
+
+    retry_max_delay : float, optional
+        Maximum randomized delay, in seconds, before retrying after a Redis
+        WATCH conflict or Redis timeout.
+        Defaults to 0.05.
+
+    socket_timeout : int, optional
+        Redis socket read timeout in seconds.
+        Defaults to None.
+
+    socket_connect_timeout : int, optional
+        Redis socket connection timeout in seconds.
+        Defaults to None.
+
+    Returns
+    -------
+    None
+        The Redis configuration is stored globally for realtime dataset
+        updates.
+
+    Raises
+    ------
+    ValueError
+        If required Redis configuration values are invalid.
+    """
+    global _redis_config
+    global _redis_client
+
+    if not isinstance(host, str) or not host.strip():
+        raise ValueError("host must be a non-empty string.")
+
+    if not isinstance(port, int) or port < 1:
+        raise ValueError("port must be a positive integer.")
+
+    if not isinstance(db, int) or db < 0:
+        raise ValueError("db must be zero or a positive integer.")
+
+    if not isinstance(key_prefix, str) or not key_prefix.strip():
+        raise ValueError("key_prefix must be a non-empty string.")
+
+    if not isinstance(max_retries, int) or max_retries < 1:
+        raise ValueError("max_retries must be a positive integer.")
+
+    if not isinstance(timeout_max_retries, int) or timeout_max_retries < 1:
+        raise ValueError("timeout_max_retries must be a positive integer.")
+
+    if not isinstance(retry_min_delay, (int, float)) or not isinstance(retry_max_delay, (int, float)):
+        raise ValueError("retry delays must be numeric.")
+
+    if retry_min_delay < 0 or retry_max_delay < 0:
+        raise ValueError("retry delays must be zero or positive.")
+
+    if retry_min_delay > retry_max_delay:
+        raise ValueError("retry_min_delay cannot be greater than retry_max_delay.")
+
+    _redis_config = {
+        "host": host,
+        "port": port,
+        "username": username,
+        "password": password,
+        "ssl": ssl,
+        "db": db,
+        "key_prefix": key_prefix,
+        "max_retries": max_retries,
+        "timeout_max_retries": timeout_max_retries,
+        "retry_min_delay": retry_min_delay,
+        "retry_max_delay": retry_max_delay,
+        "socket_timeout": socket_timeout,
+        "socket_connect_timeout": socket_connect_timeout,
+    }
+    _redis_client = None
+    logger.info(
+        f"Configured Redis realtime state. host={host}, port={port}, "
+        f"ssl={ssl}, db={db}, key_prefix={key_prefix}, "
+        f"max_retries={max_retries}, timeout_max_retries={timeout_max_retries}, "
+        f"retry_min_delay={retry_min_delay}, retry_max_delay={retry_max_delay}"
+    )
+
+
+def get_realtime_redis_config():
+    """
+    Return the current Redis realtime state configuration.
+    """
+    return dict(_redis_config)
+
+
+def is_realtime_redis_configured():
+    """
+    Return True when Redis realtime state has been configured.
+    """
+    return bool(_redis_config["host"])
+
+
+def get_realtime_redis_client():
+    """
+    Return a cached redis-py client using the configured realtime state settings.
+    """
+    global _redis_client
+
+    if not is_realtime_redis_configured():
+        raise ValueError("Redis realtime state is not configured.")
+
+    if _redis_client is None:
+        redis, _, _ = _get_redis_modules()
+
+        logger.info(
+            f"Creating Redis realtime client. host={_redis_config['host']}, "
+            f"port={_redis_config['port']}, ssl={_redis_config['ssl']}, "
+            f"db={_redis_config['db']}"
+        )
+        _redis_client = redis.Redis(
+            host=_redis_config["host"],
+            port=_redis_config["port"],
+            username=_redis_config["username"],
+            password=_redis_config["password"],
+            ssl=_redis_config["ssl"],
+            db=_redis_config["db"],
+            decode_responses=True,
+            socket_timeout=_redis_config["socket_timeout"],
+            socket_connect_timeout=_redis_config["socket_connect_timeout"],
+        )
+
+    return _redis_client
+
+
+def realtime_state_key(topic: str):
+    """
+    Build the Redis key used to store the authoritative dataset for a topic.
+    """
+    if not isinstance(topic, str) or not topic.strip():
+        raise ValueError("topic must be a non-empty string.")
+
+    return f"{_redis_config['key_prefix']}:{topic}"
+
+
+def _update_realtime_dataset_once(topic: str, update_func: Callable[[str], str]):
+    """
+    Atomically update the Redis dataset for a topic using WATCH/MULTI/EXEC.
+    """
+    _, WatchError, _ = _get_redis_modules()
+
+    client = get_realtime_redis_client()
+    key = realtime_state_key(topic)
+    max_retries = _redis_config["max_retries"]
+    logger.info(
+        f"Starting Redis realtime dataset update. topic={topic}, key={key}, "
+        f"max_retries={max_retries}"
+    )
+
+    for attempt in range(1, max_retries + 1):
+        try:
+            with client.pipeline() as pipe:
+                logger.info(
+                    f"Watching Redis realtime key={key} "
+                    f"(attempt {attempt}/{max_retries})"
+                )
+                pipe.watch(key)
+                existing_json = pipe.get(key)
+                _log_json_value(f"Redis current cache for key={key}", existing_json)
+                current_json = existing_json if existing_json else "[]"
+                updated_json = update_func(current_json)
+                if not isinstance(updated_json, str):
+                    raise ValueError("update_func must return a JSON string.")
+
+                _log_json_value(f"Redis value to set for key={key}", updated_json)
+                pipe.multi()
+                pipe.set(key, updated_json)
+                pipe.execute()
+
+                latest_json = client.get(key)
+                logger.info(f"Updated Redis realtime state for key={key}")
+                _log_json_value(f"Redis cache after update for key={key}", latest_json)
+                return updated_json
+
+        except WatchError:
+            if attempt >= max_retries:
+                break
+
+            delay = random.uniform(
+                _redis_config["retry_min_delay"],
+                _redis_config["retry_max_delay"],
+            )
+            logger.info(
+                f"Redis realtime state changed during update for {key}. "
+                f"Retrying attempt {attempt + 1}/{max_retries} after {delay:.3f}s."
+            )
+            time.sleep(delay)
+
+    raise RuntimeError(
+        f"Failed to update Redis realtime state after {max_retries} retries "
+        f"for topic: {topic}"
+    )
+
+
+def update_realtime_dataset(
+    topic: str,
+    update_func: Callable[[str], str],
+):
+    """
+    Retry Redis realtime updates when redis-py raises a timeout error.
+    """
+    key = realtime_state_key(topic)
+    timeout_max_retries = _redis_config["timeout_max_retries"]
+    last_error = None
+
+    for attempt in range(1, timeout_max_retries + 1):
+        try:
+            logger.info(
+                f"Starting Redis timeout-protected update for key={key} "
+                f"(attempt {attempt}/{timeout_max_retries})"
+            )
+            return _update_realtime_dataset_once(topic=topic, update_func=update_func)
+
+        except Exception as e:
+            if not _is_redis_timeout_error(e):
+                raise
+
+            last_error = e
+            logger.error(
+                f"Redis timeout during realtime update for key={key} "
+                f"(attempt {attempt}/{timeout_max_retries}): {e}"
+            )
+            _reset_redis_client()
+
+            if attempt >= timeout_max_retries:
+                break
+
+            delay = random.uniform(
+                _redis_config["retry_min_delay"],
+                _redis_config["retry_max_delay"],
+            )
+            logger.info(
+                f"Retrying Redis realtime update for key={key} after "
+                f"{delay:.3f}s due to timeout."
+            )
+            time.sleep(delay)
+
+    raise RuntimeError(
+        f"Redis realtime update timed out after {timeout_max_retries} attempts "
+        f"for topic: {topic}. Last error: {last_error}"
+    ) from last_error