flexCommunicator 0.1.0__py3-none-any.whl

flexCommunicator/clientLibraries/flcpy/flexCloud/flexCloudConfiguration.py

@@ -0,0 +1,16 @@
+#**************************************************************************
+# * Copyright (C) 2025-present Bert Van Acker (B.MKR) <bva.bmkr@gmail.com>
+# *
+# * This file is part of the flexIA project.
+# *
+# * flexIA can not be copied and/or distributed without the express
+# * permission of Bert Van Acker
+# *************************************************************************
+
+from flexCommunicator.clientLibraries.flcpy.utils.constants import *
+from flexCommunicator.clientLibraries.flcpy.flexCloud.flexCloudVariable import flexCloudVariable
+
+
+class flexCloudConfigParameter(flexCloudVariable):
+    def __init__(self, name, initial_value=None, syncType="onchange", sync_interval=None):
+        super().__init__(name, initial_value=initial_value, syncType=syncType, sync_interval=sync_interval,history=False,statusVariable=False,configVariable = True)

flexCommunicator/clientLibraries/flcpy/flexCloud/flexCloudManager.py

@@ -0,0 +1,162 @@
+#**************************************************************************
+# * Copyright (C) 2025-present Bert Van Acker (B.MKR) <bva.bmkr@gmail.com>
+# *
+# * This file is part of the flexIA project.
+# *
+# * flexIA can not be copied and/or distributed without the express
+# * permission of Bert Van Acker
+# *************************************************************************
+import json
+import time
+import string
+import random
+import threading
+
+from flexCommunicator.clientLibraries.flcpy.utils.constants import *
+
+# Constants for configuration keys
+KEY_KNOWLEDGE_TYPE = "knowledge_type"
+KEY_HOST = "host"
+KEY_PORT = "port"
+KEY_DB = "db"
+KEY_TOPIC = "topic"
+KEY_BOOTSTRAP_SERVERS = "bootstrap_servers"
+
+# Constants for backends
+BACKEND_REDIS = "redis"
+BACKEND_MEMCACHED = "memcached"
+BACKEND_IGNITE = "ignite"
+BACKEND_HAZELCAST = "hazelcast"
+BACKEND_TARANTOOL = "tarantool"
+BACKEND_AEROSPIKE = "aerospike"
+BACKEND_KAFKA = "kafka"
+BACKEND_SQLITE = "sqlite"
+
+# Constants for SQLite queries and in-memory DB
+SQLITE_MEMORY = ":memory:"
+SQLITE_CREATE_TABLE = "CREATE TABLE kv (key TEXT PRIMARY KEY, value TEXT)"
+SQLITE_INSERT_OR_REPLACE = "INSERT OR REPLACE INTO kv (key, value) VALUES (?, ?)"
+SQLITE_SELECT_VALUE = "SELECT value FROM kv WHERE key=?"
+SQLITE_DELETE_KEY = "DELETE FROM kv WHERE key=?"
+
+# Other constants
+ERROR_UNSUPPORTED_BACKEND = "Unsupported backend: {}"
+DEFAULT_CACHE_NAME = "default"
+AEROSPIKE_VALUE_FIELD = "value"
+
+# Helper function to generate random strings (if needed)
+def random_string(length=10):
+    return ''.join(random.choice(string.ascii_letters) for _ in range(length))
+
+
+class flexCloudManager:
+    def __init__(self, config,autostart=False):
+        """
+        Initialize the flexCloudManager with a chosen backend.
+        Supported backends: redis
+
+        :param config: A dictionary containing all configuration parameters needed by the backend.
+        """
+        self.backend = config[KEY_KNOWLEDGE_TYPE]
+        #SUBSCRIPTION VARIABLES
+        self.subscribe_topics = []
+        self.event_callbacks = {}  # Mapping: topic -> callback function
+        self._consumer_thread = None
+        #BACKEND CONFIGURATION
+        if self.backend == BACKEND_REDIS:
+            import redis
+            self.client = redis.Redis(host=config[KEY_HOST], port=config[KEY_PORT], db=config[KEY_DB])
+            self.pubsub = self.client.pubsub()
+        else:
+            raise ValueError(ERROR_UNSUPPORTED_BACKEND.format(self.backend))
+
+        #AUTO-START SUBSCRIPTON THREAD
+        if autostart:
+            self._consumer_thread = threading.Thread(target=self._listen, daemon=True)
+            self._consumer_thread.start()
+
+    def write(self, key, value):
+        """Write a key-value pair to the backend."""
+        if self.backend == BACKEND_REDIS:
+            return self.client.set(key, value)
+
+
+    def read(self, key, queueSize=1):
+        """Read the value for a given key from the backend."""
+        if self.backend == BACKEND_REDIS:
+            return self.client.get(key)
+
+    def write_history(self, key, value):
+        """
+        Append a value to the history list for `key` in Redis.
+        Returns the new length of the list.
+        """
+        if self.backend == BACKEND_REDIS:
+            # RPUSH adds to the tail of the list, creating it if needed
+            return self.client.rpush(key, value)
+
+    def read_history(self, key, history_size=1):
+        """
+        Read up to `history_size` most recent values for `key` from Redis.
+        Returns a list of values (oldest→newest within that window).
+        """
+        if self.backend == BACKEND_REDIS:
+            # Use a negative start index to grab the last `history_size` elements
+            start = -history_size
+            end = -1
+            return self.client.lrange(key, start, end)
+
+    def publish(self,key,value):
+        try:
+            data = json.dumps(value).encode('utf-8')
+        except:
+            data = value
+        self.client.publish(channel=key, message=data)
+
+    def delete(self, key):
+        """Delete the key-value pair identified by key from the backend."""
+        if self.backend == BACKEND_REDIS:
+            return self.client.delete(key)
+
+
+    def exists(self, key):
+        """Check whether a key exists in the backend."""
+        if self.backend == BACKEND_REDIS:
+            return self.client.exists(key)
+
+
+    def subscribe(self, topic, callback):
+        """Register a callback for the given topic."""
+        if topic not in self.subscribe_topics:
+            self.subscribe_topics.append(topic)
+        self.event_callbacks[topic] = callback
+
+        if self.backend == TRANSPORT_PROTOCOLS.PROTOCOL_REDIS and self.pubsub:
+            self.pubsub.subscribe(topic)
+        else:
+            raise ValueError(ERROR_UNSUPPORTED_BACKEND.format(self.backend))
+    def start(self):
+        """Start the listener thread."""
+        self._consumer_thread = threading.Thread(target=self._listen, daemon=True)
+        self._consumer_thread.start()
+
+    def stop(self):
+        """Stop the listener thread."""
+        self._consumer_thread.join(timeout=2)
+        self.pubsub.close()
+
+    #-----------------------helper functions------------------------------
+    def _listen(self):
+        try:
+            for message in self.pubsub.listen():
+                if message['type'] == 'message':
+                    topic = message['channel'].decode('utf-8')              #TODO: check if needed
+                    if topic in self.event_callbacks:
+                        message_decoded = message['data'].decode('utf-8')
+                        self.event_callbacks[topic](message_decoded)        #TODO: check if needed
+        except (ValueError, OSError) as e:
+            # Socket was likely closed during shutdown; exit gracefully.
+            print(FLEXCLOUD_ERRORS.ERR_REDIS_SUBSCRIPTION_CLOSED.format(e))
+
+
+

flexCommunicator/clientLibraries/flcpy/flexCloud/flexCloudOrchestrator.py

@@ -0,0 +1,248 @@
+#**************************************************************************
+# * Copyright (C) 2025-present Bert Van Acker (B.MKR) <bva.bmkr@gmail.com>
+# *
+# * This file is part of the flexIA project.
+# *
+# * flexIA can not be copied and/or distributed without the express
+# * permission of Bert Van Acker
+# *************************************************************************
+from __future__ import annotations
+
+import time
+import threading
+from dataclasses import dataclass
+from enum import Enum
+from typing import Callable, Dict, Iterable, List, Optional, Sequence, Set, Tuple
+
+import redis
+
+from flexCommunicator.clientLibraries.flcpy.utils.constants import *
+
+@dataclass(frozen=True)
+class ComponentSpec:
+    """
+    name: unique component identifier (used for Redis keying)
+    spin: callable that starts the component
+    depends_on: optional list of component names that must be RUNNING first
+    """
+    name: str
+    spin: Callable[[], None]
+    depends_on: Tuple[str, ...] = ()
+
+
+class StartupError(RuntimeError):
+    pass
+
+
+class StartupOrchestrator:
+    """
+    Orchestrates startup of Python components that expose state in Redis.
+
+    Default Redis key pattern: "{name}:state" with value "RUNNING", etc.
+    """
+
+    def __init__(
+        self,
+        components: Sequence[ComponentSpec],
+        *,
+        # Redis connection settings (integrated)
+        redis_url: Optional[str] = None,
+        host: str = "localhost",
+        port: int = 6379,
+        db: int = 6,
+        username: Optional[str] = None,
+        password: Optional[str] = None,
+        ssl: bool = False,
+        socket_timeout_s: float = 2.0,
+        decode_responses: bool = True,
+        health_check_interval_s: int = 10,
+        # Orchestration settings
+        connectorID: str = "appName@projectName-key",
+        running_value: str =  APPLICATION_STATUS.RUNNING,
+        poll_interval_s: float = 0.1,
+        spin_in_thread: bool = True,
+        ping_on_init: bool = True,
+    ) -> None:
+        self.components: Dict[str, ComponentSpec] = {c.name: c for c in components}
+
+        # Orchestration config
+        self.key_pattern = connectorID + ":applications:{name}:status"
+        self.running_value = running_value
+        self.poll_interval_s = poll_interval_s
+        self.spin_in_thread = spin_in_thread
+
+        # Build Redis client (integrated)
+        if redis_url:
+            self.r = redis.Redis.from_url(
+                redis_url,
+                socket_timeout=socket_timeout_s,
+                decode_responses=decode_responses,
+                health_check_interval=health_check_interval_s,
+            )
+        else:
+            self.r = redis.Redis(
+                host=host,
+                port=port,
+                db=db,
+                username=username,
+                password=password,
+                ssl=ssl,
+                socket_timeout=socket_timeout_s,
+                decode_responses=decode_responses,
+                health_check_interval=health_check_interval_s,
+            )
+
+        if ping_on_init:
+            try:
+                self.r.ping()
+            except Exception as e:
+                raise StartupError(f"Redis ping failed (check connection settings): {e}") from e
+
+        # Validate dependencies reference known components
+        for c in components:
+            for dep in c.depends_on:
+                if dep not in self.components:
+                    raise ValueError(f"Component '{c.name}' depends on unknown component '{dep}'")
+
+    def close(self) -> None:
+        """
+        Close the Redis connection pool cleanly.
+        """
+        try:
+            self.r.close()
+        except Exception:
+            # redis-py versions differ; best-effort close
+            try:
+                self.r.connection_pool.disconnect()
+            except Exception:
+                pass
+
+    def _key_for(self, name: str) -> str:
+        return self.key_pattern.format(name=name)
+
+    def _read_state(self, name: str) -> str:
+        try:
+            val = self.r.get(self._key_for(name))
+        except Exception as e:
+            raise StartupError(f"Redis GET failed for '{self._key_for(name)}': {e}") from e
+
+        if val is None:
+            return APPLICATION_STATUS.UNKNOWN
+        # if decode_responses=False, val is bytes; else it's str
+        if isinstance(val, bytes):
+            return val.decode("utf-8", errors="replace")
+        return str(val)
+
+    def _wait_for_running(self, name: str, timeout_s: float) -> None:
+        print(f"Waiting for '{name}' to become {self.running_value}...")
+        deadline = time.time() + timeout_s
+        while True:
+            state = self._read_state(name)
+            if state == self.running_value:
+                print(f"'{name}' is now {self.running_value}.")
+                return
+            if state == APPLICATION_STATUS.ERROR:
+                raise StartupError(f"Component '{name}' entered ERROR state while starting")
+
+            if time.time() >= deadline:
+                raise StartupError(
+                    f"Timeout waiting for '{name}' to become {self.running_value}. "
+                    f"Last state: {state!r}, key: {self._key_for(name)!r}"
+                )
+            time.sleep(self.poll_interval_s)
+
+
+    def _topological_order(self, subset: Optional[Iterable[str]] = None) -> List[str]:
+        if subset is None:
+            nodes: Set[str] = set(self.components.keys())
+        else:
+            nodes = set(subset)
+
+        # Ensure requested subset includes its dependencies
+        missing_edges = []
+        for n in nodes:
+            for d in self.components[n].depends_on:
+                if d not in nodes:
+                    missing_edges.append((n, d))
+        if missing_edges:
+            pairs = ", ".join([f"{n}->{d}" for n, d in sorted(missing_edges)])
+            raise ValueError("Subset does not include all dependencies. Missing edges: " + pairs)
+
+        indeg: Dict[str, int] = {n: 0 for n in nodes}
+        adj: Dict[str, List[str]] = {n: [] for n in nodes}
+
+        for n in nodes:
+            for d in self.components[n].depends_on:
+                adj[d].append(n)   # d -> n
+                indeg[n] += 1
+
+        q = sorted([n for n, deg in indeg.items() if deg == 0])
+        out: List[str] = []
+
+        while q:
+            n = q.pop(0)
+            out.append(n)
+            for m in adj[n]:
+                indeg[m] -= 1
+                if indeg[m] == 0:
+                    q.append(m)
+                    q.sort()
+
+        if len(out) != len(nodes):
+            remaining = [n for n in nodes if n not in out]
+            raise ValueError(f"Dependency cycle detected among: {sorted(remaining)}")
+
+        return out
+
+    def _start_one(self, name: str) -> None:
+        spec = self.components[name]
+
+        def runner():
+            spec.spin()
+
+        if self.spin_in_thread:
+            threading.Thread(target=runner, name=f"spin:{name}", daemon=True).start()
+        else:
+            runner()
+
+    def startup(
+        self,
+        *,
+        order: Optional[Sequence[str]] = None,
+        subset: Optional[Iterable[str]] = None,
+        timeout_per_component_s: float = 30.0,
+        fail_fast: bool = True,
+        ensure_not_running: bool = False,
+    ) -> List[str]:
+        if order is None:
+            start_order = self._topological_order(subset=subset)
+        else:
+            start_order = list(order)
+
+        failures: List[str] = []
+
+        for name in start_order:
+            if name not in self.components:
+                raise ValueError(f"Unknown component in order: '{name}'")
+
+            current = self._read_state(name)
+            if ensure_not_running and current == self.running_value:
+                msg = f"Component '{name}' is already {self.running_value}"
+                if fail_fast:
+                    raise StartupError(msg)
+                failures.append(msg)
+                continue
+
+            self._start_one(name)
+            try:
+                self._wait_for_running(name, timeout_s=timeout_per_component_s)
+            except Exception as e:
+                msg = f"Failed to start '{name}': {e}"
+                if fail_fast:
+                    raise StartupError(msg) from e
+                failures.append(msg)
+
+        if failures:
+            raise StartupError("One or more components failed:\n- " + "\n- ".join(failures))
+
+        return start_order

flexCommunicator/clientLibraries/flcpy/flexCloud/flexCloudVariable.py

@@ -0,0 +1,130 @@
+#**************************************************************************
+# * Copyright (C) 2025-present Bert Van Acker (B.MKR) <bva.bmkr@gmail.com>
+# *
+# * This file is part of the flexIA project.
+# *
+# * flexIA can not be copied and/or distributed without the express
+# * permission of Bert Van Acker
+# *************************************************************************
+import threading
+import json
+import time
+
+from flexCommunicator.clientLibraries.flcpy.utils.constants import *
+
+
+class flexCloudVariable(object):
+    def __init__(self, name, initial_value=0, syncType="onchange", sync_interval=None,history=True,statusVariable=False,configVariable = False):
+        self._name = name
+        self._value = initial_value
+        self._lock = threading.Lock()
+
+
+        #flexCloud handler
+        self.history = history
+        self.flexCloud_manager = None
+        self.flexCloud_connectorID = None
+        self.flexCloud_applicationID = None
+
+        # flexCloud application status
+        self.statusVariable = statusVariable
+        if statusVariable:
+            self.history = False
+        # flexCloud config variable
+        self.configVariable = configVariable
+        if configVariable:
+            self.history = False
+
+
+    def _sync_to_redis(self):
+        if self.statusVariable:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + ":"+ self._name
+        elif self.configVariable:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + FLEXCLOUD_VARIABLES.CONFIGURATION + self._name
+        else:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + FLEXCLOUD_VARIABLES.VARIABLES + self._name
+        self._write_knowledge(key=key, value=self._value)
+
+    def _retrieve_from_redis(self,queueSize=1):
+        if self.statusVariable:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + ":"+ self._name
+        elif self.configVariable:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + FLEXCLOUD_VARIABLES.CONFIGURATION + self._name
+        else:
+            key = self.flexCloud_connectorID + FLEXCLOUD_VARIABLES.APPLICATIONS + self.flexCloud_applicationID + FLEXCLOUD_VARIABLES.VARIABLES + self._name
+        return self._read_knowledge(key=key,queueSize=queueSize,decode=True)       #TODO: assuming that variables are always stored raw
+
+    def initialize(self):
+        """ Initialize the flexCloud variable"""
+        self._sync_to_redis()
+
+    def set(self, value):
+        if value != self._value:
+            self._value = value
+            self._sync_to_redis()
+
+    def get(self,queueSize=1):
+        with self._lock:
+            # -- variable with history --
+            if self.history:
+                queue = self._retrieve_from_redis(queueSize=queueSize)
+                if len(queue) == 1:
+                    self._value = queue[0]      #AUTO-SET VALUE
+                    return queue[0]
+                else:
+                    return queue
+            # -- variable without history --
+            else:
+                value = self._retrieve_from_redis(queueSize=1)
+                self._value = value     #AUTO-SET VALUE
+                return value
+
+    def fetch_from_redis(self):                                             #TODO: this needs to be checked periodically for changes in the redis if variable is read/write!
+        value = self._read_knowledge(key=self._name)
+        with self._lock:
+            self._value = value
+        return value
+
+
+
+    # --------------------------- HELPER FUNCTIONS ------------------------
+    def _write_knowledge(self, key, value):
+        """Write a value to the knowledge Manager."""
+        if isinstance(key, str):
+            if isinstance(value, str):
+                value = str(value)
+            if self.history:
+                return self.flexCloud_manager.write_history(key, value)
+            else:
+                return self.flexCloud_manager.write(key, value)
+        else:
+            # Convert the class instance to a dictionary
+            class_dict = {}
+            for key_attr, attr_value in key.__dict__.items():
+                public_key = key_attr.lstrip('_')
+                class_dict[public_key] = attr_value
+            value = json.dumps(class_dict)  # Serialize the dictionary to a JSON string
+            if self.history:
+                return self.flexCloud_manager.write_history(key.name, value)
+            else:
+                return self.flexCloud_manager.write(key.name, value)
+
+    def _read_knowledge(self, key, queueSize=1, decode=True):
+        """Read a value from the knowledge Manager."""
+        # -- variable with history --
+        if self.history:
+            value = self.flexCloud_manager.read_history(key, queueSize)
+            print(value)
+            if decode:
+                if value is not None:
+                    # decode all values of array
+                    for i in range (0,len(value),1):
+                        value[i] = value[i].decode(ENCODING.ENCODING_UTF8)
+            return value
+
+        # -- variable without history --
+        else:
+            value = self.flexCloud_manager.read(key)
+            if decode:
+                value = value.decode(ENCODING.ENCODING_UTF8)
+            return value