dbus2mqtt 0.1.0__py3-none-any.whl

dbus2mqtt/event_broker.py

@@ -0,0 +1,70 @@
+import asyncio
+import logging
+
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Any
+
+import janus
+
+from dbus2mqtt.config import (
+    FlowConfig,
+    FlowTriggerConfig,
+    SignalConfig,
+    SubscriptionConfig,
+)
+from dbus2mqtt.dbus.dbus_types import BusNameSubscriptions
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class MqttMessage:
+    topic: str
+    payload: Any
+    payload_serialization_type: str = "json"
+
+@dataclass
+class DbusSignalWithState:
+    bus_name_subscriptions: BusNameSubscriptions
+    path: str
+    interface_name: str
+    subscription_config: SubscriptionConfig
+    signal_config: SignalConfig
+    args: list[Any]
+
+@dataclass
+class FlowTriggerMessage:
+    flow_config: FlowConfig
+    flow_trigger_config: FlowTriggerConfig
+    timestamp: datetime
+    context: dict[str, Any] | None = None
+
+class EventBroker:
+    def __init__(self):
+        self.mqtt_receive_queue = janus.Queue[MqttMessage]()
+        self.mqtt_publish_queue = janus.Queue[MqttMessage]()
+        self.dbus_signal_queue = janus.Queue[DbusSignalWithState]()
+        self.flow_trigger_queue = janus.Queue[FlowTriggerMessage]()
+        # self.dbus_send_queue: janus.Queue
+
+    async def close(self):
+        await asyncio.gather(
+            self.mqtt_receive_queue.aclose(),
+            self.mqtt_publish_queue.aclose(),
+            self.dbus_signal_queue.aclose(),
+            self.flow_trigger_queue.aclose(),
+            return_exceptions=True
+        )
+
+    def on_mqtt_receive(self, msg: MqttMessage):
+        # logger.debug("on_mqtt_receive")
+        self.mqtt_receive_queue.sync_q.put(msg)
+
+    async def publish_to_mqtt(self, msg: MqttMessage):
+        # logger.debug("publish_to_mqtt")
+        await self.mqtt_publish_queue.async_q.put(msg)
+
+    def on_dbus_signal(self, signal: DbusSignalWithState):
+        # logger.debug("on_dbus_signal")
+        self.dbus_signal_queue.sync_q.put(signal)

dbus2mqtt/flow/__init__.py

@@ -0,0 +1,32 @@
+from abc import ABC, abstractmethod
+from typing import Any
+
+
+class FlowExecutionContext:
+
+    def __init__(self, name: str | None, global_flows_context: dict[str, Any], flow_context: dict[str, Any]):
+        self.name = name
+        self.global_flows_context = global_flows_context
+        self.flow_context = flow_context
+
+        # per flow execution context
+        self.context: dict[str, Any] = {}
+
+    def get_aggregated_context(self) -> dict[str, Any]:
+        """Get the aggregated context for the flow execution."""
+        # Merge global flows context, flow context, and local context
+        context = {}
+        if self.global_flows_context:
+            context.update(self.global_flows_context)
+        if self.flow_context:
+            context.update(self.flow_context)
+        if self.context:
+            context.update(self.context)
+        return context
+
+class FlowAction(ABC):
+
+    @abstractmethod
+    async def execute(self, context: FlowExecutionContext):
+        """Execute the action with the given flow execution context."""
+        pass

dbus2mqtt/flow/actions/context_set.py

@@ -0,0 +1,26 @@
+import logging
+
+from dbus2mqtt import AppContext
+from dbus2mqtt.config import FlowActionContextSetConfig
+from dbus2mqtt.flow import FlowAction, FlowExecutionContext
+
+logger = logging.getLogger(__name__)
+
+class ContextSetAction(FlowAction):
+
+    def __init__(self, config: FlowActionContextSetConfig, app_context: AppContext):
+        self.config = config
+        self.templating = app_context.templating
+
+    async def execute(self, context: FlowExecutionContext):
+
+        aggregated_context = context.get_aggregated_context()
+        if self.config.global_context:
+            context_new = await self.templating.async_render_template(self.config.global_context, dict, aggregated_context)
+            logger.debug(f"Update global_context with: {context_new}")
+            context.global_flows_context.update(context_new)
+
+        if self.config.context:
+            context_new = await self.templating.async_render_template(self.config.context, dict, aggregated_context)
+            logger.debug(f"Update context with: {context_new}")
+            context.context.update(context_new)

dbus2mqtt/flow/actions/mqtt_publish.py

@@ -0,0 +1,39 @@
+
+import logging
+
+from jinja2.exceptions import TemplateRuntimeError
+
+from dbus2mqtt import AppContext
+from dbus2mqtt.config import FlowActionMqttPublishConfig
+from dbus2mqtt.flow import FlowAction, FlowExecutionContext
+from dbus2mqtt.mqtt.mqtt_client import MqttMessage
+
+logger = logging.getLogger(__name__)
+
+class MqttPublishAction(FlowAction):
+
+    def __init__(self, config: FlowActionMqttPublishConfig, app_context: AppContext):
+        self.config = config
+        self.event_broker = app_context.event_broker
+        self.templating = app_context.templating
+
+    async def execute(self, context: FlowExecutionContext):
+
+        render_context = context.get_aggregated_context()
+
+        try:
+            mqtt_topic = await self.templating.async_render_template(self.config.topic, str, render_context)
+
+            payload_res_type = str if self.config.payload_type == "text" else dict
+            payload = await self.templating.async_render_template(self.config.payload_template, payload_res_type, render_context)
+
+        except TemplateRuntimeError as e:
+            logger.warning(f"Error rendering jinja template, flow: '{context.name}', error: {str(e)}. render_context={render_context}", exc_info=True)
+            return
+        except Exception as e:
+            logger.warning(f"Error rendering jinja template, flow: '{context.name}', error: {str(e)}. render_context={render_context}", exc_info=False)
+            return
+
+        logger.debug(f"public_mqtt: flow={context.name}, payload={payload}")
+
+        await self.event_broker.publish_to_mqtt(MqttMessage(mqtt_topic, payload, payload_serialization_type=self.config.payload_type))

dbus2mqtt/flow/flow_processor.py

@@ -0,0 +1,197 @@
+import asyncio
+import logging
+
+from datetime import datetime
+from typing import Any
+
+from apscheduler.schedulers.asyncio import AsyncIOScheduler
+
+from dbus2mqtt import AppContext
+from dbus2mqtt.config import FlowConfig, FlowTriggerConfig
+from dbus2mqtt.event_broker import FlowTriggerMessage
+from dbus2mqtt.flow import FlowAction, FlowExecutionContext
+from dbus2mqtt.flow.actions.context_set import ContextSetAction
+from dbus2mqtt.flow.actions.mqtt_publish import MqttPublishAction
+
+logger = logging.getLogger(__name__)
+
+class FlowScheduler:
+
+    def __init__(self, app_context: AppContext):
+        self.config = app_context.config
+        self.event_broker = app_context.event_broker
+        self.scheduler = AsyncIOScheduler()
+
+    async def _schedule_flow_strigger(self, flow, trigger_config: FlowTriggerConfig):
+        trigger = FlowTriggerMessage(flow, trigger_config, datetime.now())
+        await self.event_broker.flow_trigger_queue.async_q.put(trigger)
+
+    async def scheduler_task(self):
+
+        self.scheduler.start()
+
+        # configure global flow trigger
+        self.start_flow_set(self.config.flows)
+
+        while True:
+            await asyncio.sleep(1000)
+
+    def start_flow_set(self, flows: list[FlowConfig]):
+        for flow in flows:
+            for trigger in flow.triggers:
+                if trigger.type == "schedule":
+                    existing_job = self.scheduler.get_job(trigger.id)
+                    if existing_job:
+                        logger.debug(f"Skipping creation, flow scheduler already exists, id={trigger.id}")
+                    if not existing_job and trigger.type == "schedule":
+                        logger.info(f"Starting scheduler[{trigger.id}] for flow {flow.id}")
+                        if trigger.interval:
+                            # Each schedule gets its own job
+                            self.scheduler.add_job(
+                                self._schedule_flow_strigger,
+                                "interval",
+                                id=trigger.id,
+                                max_instances=1,
+                                misfire_grace_time=5,
+                                coalesce=True,
+                                args=[flow, trigger],
+                                **trigger.interval
+                            )
+                        elif trigger.cron:
+                            # Each schedule gets its own job
+                            self.scheduler.add_job(
+                                self._schedule_flow_strigger,
+                                "cron",
+                                id=trigger.id,
+                                max_instances=1,
+                                misfire_grace_time=5,
+                                coalesce=True,
+                                args=[flow, trigger],
+                                **trigger.cron
+                            )
+
+    def stop_flow_set(self, flows):
+        for flow in flows:
+            for trigger in flow.triggers:
+                if trigger.type == "schedule":
+                    logger.info(f"Stopping scheduler[{trigger.id}] for flow {flow.id}")
+                    self.scheduler.remove_job(trigger.id)
+
+class FlowActionContext:
+
+    def __init__(self, app_context: AppContext, flow_config: FlowConfig, global_flows_context: dict[str, Any], flow_context: dict[str, Any]):
+        self.app_context = app_context
+        self.global_flows_context = global_flows_context
+        self.flow_context = flow_context
+        self.flow_config = flow_config
+
+        self.flow_actions = self._setup_flow_actions()
+
+    def _setup_flow_actions(self) -> list[FlowAction]:
+
+        res = []
+        for action_config in self.flow_config.actions:
+            action = None
+            if action_config.type == "context_set":
+                action = ContextSetAction(action_config, self.app_context)
+            if action_config.type == "mqtt_publish":
+                action = MqttPublishAction(action_config, self.app_context)
+            if action:
+                res.append(action)
+
+        return res
+
+    async def execute_actions(self, trigger_context: dict[str, Any] | None):
+
+        # per flow execution context
+        context = FlowExecutionContext(
+            self.flow_config.name,
+            global_flows_context=self.global_flows_context,
+            flow_context=self.flow_context)
+
+        if trigger_context:
+            context.context.update(trigger_context)
+
+        for action in self.flow_actions:
+            await action.execute(context)
+
+class FlowProcessor:
+
+    def __init__(self, app_context: AppContext):
+        self.app_context = app_context
+        self.event_broker = app_context.event_broker
+
+        self._global_context: dict[str, Any] = {}
+
+        self._flows: dict[str, FlowActionContext] = {}
+
+        # register global flows
+        self.register_flows(app_context.config.flows)
+
+        # register dbus subscription flows
+        for subscription in app_context.config.dbus.subscriptions:
+            flow_context = {
+                "subscription_bus_name": subscription.bus_name,
+                "subscription_path": subscription.path,
+                "subscription_interfaces": [i.interface for i in subscription.interfaces],
+            }
+            self.register_flows(subscription.flows, flow_context)
+
+    def register_flows(self, flows: list[FlowConfig], flow_context: dict[str, Any] = {}):
+        """Register flows with the flow processor."""
+
+        for flow_config in flows:
+            flow_action_context = FlowActionContext(
+                self.app_context,
+                flow_config,
+                self._global_context,
+                flow_context
+            )
+            self._flows[flow_config.id] = flow_action_context
+
+    async def flow_processor_task(self):
+        """Continuously processes messages from the async queue."""
+
+        # logger.info(f"flow_processor_task: configuring flows={[f.name for f in self.app_context.config.flows]}")
+
+        while True:
+            flow_trigger_message = await self.event_broker.flow_trigger_queue.async_q.get()  # Wait for a message
+            try:
+                await self._process_flow_trigger(flow_trigger_message)
+
+            except Exception as e:
+                logger.warning(f"flow_processor_task: Exception {e}", exc_info=True)
+            finally:
+                self.event_broker.flow_trigger_queue.async_q.task_done()
+
+    async def _process_flow_trigger(self, flow_trigger_message: FlowTriggerMessage):
+        log_message = f"on_trigger: {flow_trigger_message.flow_trigger_config.type}, time={flow_trigger_message.timestamp.isoformat()}"
+        if flow_trigger_message.flow_trigger_config.type != "schedule":
+            logger.info(log_message)
+        else:
+            logger.debug(log_message)
+
+        flow_id = flow_trigger_message.flow_config.id
+        # flow_name = flow_trigger_message.flow_config.name
+
+        flow = self._flows[flow_id]
+        await flow.execute_actions(trigger_context=flow_trigger_message.context)
+
+# # Create a flow from the YAML configuration
+# for flow_config in config['flows']:
+#     flow_name = flow_config['name']
+#     triggers = flow_config.get('triggers', [])
+#     actions = flow_config.get('actions', [])
+
+#     with Flow(flow_name) as flow:
+#         data = "sensor_data"
+#         for action in actions:
+#             if action['type'] == 'python_script':
+#                 process_data(data)
+#             elif action['type'] == 'mqtt_publish':
+#                 mqtt_publish(action['topic'], action['message_template'], data)
+
+#         # Add scheduling trigger if defined
+#         for trigger in triggers:
+#             if trigger['type'] == 'schedule' and 'cron' in trigger:
+#                 flow.schedule = CronSchedule(cron=trigger['cron'])

dbus2mqtt/main.py

@@ -0,0 +1,135 @@
+import asyncio
+import logging
+import sys
+
+from typing import cast
+
+import colorlog
+import dbus_next.aio as dbus_aio
+import dotenv
+import jsonargparse
+
+from dbus2mqtt import AppContext
+from dbus2mqtt.config import Config
+from dbus2mqtt.dbus.dbus_client import DbusClient
+from dbus2mqtt.event_broker import EventBroker
+from dbus2mqtt.flow.flow_processor import FlowProcessor, FlowScheduler
+from dbus2mqtt.mqtt.mqtt_client import MqttClient
+from dbus2mqtt.template.dbus_template_functions import jinja_custom_dbus_functions
+from dbus2mqtt.template.templating import TemplateEngine
+
+logger = logging.getLogger(__name__)
+
+
+async def dbus_processor_task(app_context: AppContext, flow_scheduler: FlowScheduler):
+
+    bus = dbus_aio.message_bus.MessageBus()
+
+    dbus_client = DbusClient(app_context, bus, flow_scheduler)
+    app_context.templating.add_functions(jinja_custom_dbus_functions(dbus_client))
+
+    await dbus_client.connect()
+
+    loop = asyncio.get_running_loop()
+    dbus_client_run_future = loop.create_future()
+
+    await asyncio.gather(
+        dbus_client_run_future,
+        asyncio.create_task(dbus_client.dbus_signal_queue_processor_task()),
+        asyncio.create_task(dbus_client.mqtt_receive_queue_processor_task())
+    )
+
+async def mqtt_processor_task(app_context: AppContext):
+
+    mqtt_client = MqttClient(app_context)
+
+    mqtt_client.connect()
+    mqtt_client.client.loop_start()
+
+    loop = asyncio.get_running_loop()
+    mqtt_client_run_future = loop.create_future()
+
+    try:
+        await asyncio.gather(
+            mqtt_client_run_future,
+            asyncio.create_task(mqtt_client.mqtt_publish_queue_processor_task())
+        )
+    except asyncio.CancelledError:
+        mqtt_client.client.loop_stop()
+
+async def flow_processor_task(app_context: AppContext):
+
+    flow_processor = FlowProcessor(app_context)
+
+    await asyncio.gather(
+        asyncio.create_task(flow_processor.flow_processor_task())
+    )
+
+async def run(config: Config):
+
+    event_broker = EventBroker()
+    template_engine = TemplateEngine()
+
+    app_context = AppContext(config, event_broker, template_engine)
+
+    flow_scheduler = FlowScheduler(app_context)
+
+    try:
+        await asyncio.gather(
+            dbus_processor_task(app_context, flow_scheduler),
+            mqtt_processor_task(app_context),
+            flow_processor_task(app_context),
+            asyncio.create_task(flow_scheduler.scheduler_task())
+        )
+    except asyncio.CancelledError:
+        pass
+
+def main():
+
+    # load environment from .env if it exists
+    dotenv_file = dotenv.find_dotenv(usecwd=True)
+    if len(dotenv_file) > 0:
+        logger.info(f"Loaded environment variables from {dotenv_file}")
+        dotenv.load_dotenv(dotenv_path=dotenv_file)
+
+    # unless specified otherwise, load config from config.yaml
+    parser = jsonargparse.ArgumentParser(default_config_files=["config.yaml"], default_env=True, env_prefix=False)
+
+    parser.add_argument("--verbose", "-v", nargs="?", const=True, help="Enable verbose logging")
+    parser.add_argument("--config", action="config")
+    parser.add_class_arguments(Config)
+
+    cfg = parser.parse_args()
+
+    config: Config = cast(Config, parser.instantiate_classes(cfg))
+
+    handler = colorlog.StreamHandler(stream=sys.stdout)
+    handler.setFormatter(colorlog.ColoredFormatter(
+        '%(log_color)s%(levelname)s:%(name)s:%(message)s',
+        log_colors={
+            "DEBUG": "light_black",
+            "WARNING": "yellow",
+            "ERROR": "red",
+            "CRITICAL": "bold_red",
+        }
+    ))
+
+    if cfg.verbose:
+        logging.basicConfig(level=logging.DEBUG, handlers=[handler])
+    else:
+        logging.basicConfig(level=logging.INFO, handlers=[handler])
+        apscheduler_logger = logging.getLogger("apscheduler")
+        apscheduler_logger.setLevel(logging.WARNING)
+
+
+    # handler.setFormatter(colorlog.ColoredFormatter('%(log_color)s%(levelname)s:%(name)s:%(message)s'))
+
+    # logger = colorlog.getLogger('')
+    # for handler in logger.handlers:
+    #     print(handler.st)
+    #     if isinstance(handler, colorlog.StreamHandler):
+    #         handler.setFormatter(colorlog.ColoredFormatter('%(log_color)s%(levelname)s:%(name)s:%(message)s'))
+
+    logger.debug(f"config: {config}")
+
+    asyncio.run(run(config))

dbus2mqtt/mqtt/mqtt_client.py

@@ -0,0 +1,101 @@
+
+import asyncio
+import json
+import logging
+
+from typing import Any
+
+import paho.mqtt.client as mqtt
+import yaml
+
+from paho.mqtt.enums import CallbackAPIVersion
+from paho.mqtt.subscribeoptions import SubscribeOptions
+
+from dbus2mqtt import AppContext
+from dbus2mqtt.event_broker import MqttMessage
+
+logger = logging.getLogger(__name__)
+
+class MqttClient:
+
+    def __init__(self, app_context: AppContext):
+        self.config = app_context.config.mqtt
+        self.event_broker = app_context.event_broker
+
+        self.client = mqtt.Client(
+            protocol=mqtt.MQTTv5,
+            callback_api_version=CallbackAPIVersion.VERSION2
+        )
+
+        self.client.username_pw_set(
+            username=self.config.username,
+            password=self.config.password.get_secret_value()
+        )
+
+        self.client.on_connect = self.on_connect
+        self.client.on_message = self.on_message
+
+    def connect(self):
+
+        # mqtt_client.on_message = lambda client, userdata, message: asyncio.create_task(mqtt_on_message(client, userdata, message))
+        self.client.connect_async(
+            host=self.config.host,
+            port=self.config.port
+        )
+
+    # def on_dbus_signal(self, bus_name: str, path: str, interface: str, signal: str, topic, msg: dict[str, Any]):
+    #     payload = json.dumps(msg)
+    #     logger.debug(f"on_dbus_signal: payload={payload}")
+    #     self.client.publish(topic=topic, payload=payload)
+
+    async def mqtt_publish_queue_processor_task(self):
+        """Continuously processes messages from the async queue."""
+        while True:
+            msg = await self.event_broker.mqtt_publish_queue.async_q.get()  # Wait for a message
+            try:
+                payload = msg.payload
+                type = msg.payload_serialization_type
+                if isinstance(msg.payload, dict):
+                    if type == "json":
+                        payload = json.dumps(msg.payload)
+                    elif type == "yaml":
+                        payload = yaml.dump(msg.payload)
+                elif type == "text":
+                    payload = str(payload)
+
+                logger.debug(f"mqtt_publish_queue_processor_task: payload={payload}")
+                self.client.publish(topic=msg.topic, payload=payload)
+            except Exception as e:
+                logger.warning(f"mqtt_publish_queue_processor_task: Exception {e}", exc_info=True)
+            finally:
+                self.event_broker.mqtt_publish_queue.async_q.task_done()
+
+
+    async def run(self):
+        """Runs the MQTT loop in a non-blocking way with asyncio."""
+        self.client.loop_start()  # Runs Paho's loop in a background thread
+        await asyncio.Event().wait()  # Keeps the coroutine alive
+
+    # The callback for when the client receives a CONNACK response from the server.
+    def on_connect(self, client: mqtt.Client, userdata, flags, reason_code, properties):
+        if reason_code.is_failure:
+            logger.warning(f"on_connect: Failed to connect: {reason_code}. Will retry connection")
+        else:
+            logger.info(f"on_connect: Connected to {self.config.host}:{self.config.port}")
+            # Subscribing in on_connect() means that if we lose the connection and
+            # reconnect then subscriptions will be renewed.
+            client.subscribe("dbus2mqtt/#", options=SubscribeOptions(noLocal=True))
+
+    def on_message(self, client: mqtt.Client, userdata: Any, msg: mqtt.MQTTMessage):
+
+        payload = msg.payload.decode()
+        if msg.retain:
+            logger.info(f"on_message: skipping msg with retain=True, topic={msg.topic}, payload={payload}")
+            return
+
+        try:
+            json_payload = json.loads(payload)
+            logger.debug(f"on_message: msg.topic={msg.topic}, msg.payload={json.dumps(json_payload)}")
+            self.event_broker.on_mqtt_receive(MqttMessage(msg.topic, json_payload))
+        except json.JSONDecodeError as e:
+            logger.warning(f"on_message: Unexpected payload, expecting json, topic={msg.topic}, payload={payload}, error={e}")

dbus2mqtt/template/dbus_template_functions.py

@@ -0,0 +1,68 @@
+import fnmatch
+import logging
+
+from typing import Any
+
+from dbus_next.constants import ErrorType
+from dbus_next.errors import DBusError
+
+from dbus2mqtt.dbus.dbus_client import DbusClient
+
+logger = logging.getLogger(__name__)
+
+class DbusContext:
+
+    def __init__(self, dbus_client: DbusClient):
+        self.dbus_client = dbus_client
+
+    def async_dbus_list_fn(self, bus_name_pattern: str):
+
+        res = []
+
+        self.dbus_client.subscriptions
+        for bus_name in self.dbus_client.subscriptions.keys():
+            if fnmatch.fnmatchcase(bus_name, bus_name_pattern):
+                res.append(bus_name)
+
+        return res
+
+    async def async_dbus_call_fn(self, bus_name: str, path: str, interface: str, method:str, method_args: list[Any] = []):
+
+        if not isinstance(method_args, list):
+            # Pylance will mentiod this line is unreachable. It is not as jinja2 can pass in any type
+            raise ValueError("method_args must be a list")
+
+        proxy_object = self.dbus_client.get_proxy_object(bus_name, path)
+        if not proxy_object:
+            raise ValueError(f"No matching subscription found for bus_name: {bus_name}, path: {path}")
+
+        obj_interface = proxy_object.get_interface(interface)
+
+        return await self.dbus_client.call_dbus_interface_method(obj_interface, method, method_args)
+
+    async def async_dbus_property_get_fn(self, bus_name: str, path: str, interface: str, property:str, default_unsupported: Any = None):
+
+        proxy_object = self.dbus_client.get_proxy_object(bus_name, path)
+        if not proxy_object:
+            raise ValueError(f"No matching subscription found for bus_name: {bus_name}, path: {path}")
+
+        obj_interface = proxy_object.get_interface(interface)
+
+        try:
+            return await self.dbus_client.get_dbus_interface_property(obj_interface, property)
+        except DBusError as e:
+            if e.type == ErrorType.NOT_SUPPORTED.value and default_unsupported is not None:
+                return default_unsupported
+
+def jinja_custom_dbus_functions(dbus_client: DbusClient) -> dict[str, Any]:
+
+    dbus_context = DbusContext(dbus_client)
+
+    custom_functions: dict[str, Any] = {}
+    custom_functions.update({
+        "dbus_list": dbus_context.async_dbus_list_fn,
+        "dbus_call": dbus_context.async_dbus_call_fn,
+        "dbus_property_get": dbus_context.async_dbus_property_get_fn
+    })
+
+    return custom_functions