pepeunit-client 0.10.0__py3-none-any.whl

pepeunit_client/__init__.py

@@ -0,0 +1,11 @@
+"""
+PepeunitClient - Мультиплатформенная библиотека для работы с Pepeunit Unit Storage
+"""
+
+__version__ = "0.10.0"
+
+from .client import PepeunitClient
+from .abstract_clients import AbstractPepeunitMqttClient, AbstractPepeunitRestClient
+from .enums import RestartMode
+
+__all__ = ['PepeunitClient', 'AbstractPepeunitMqttClient', 'AbstractPepeunitRestClient', 'RestartMode']

pepeunit_client/abstract_clients.py

@@ -0,0 +1,156 @@
+from abc import ABC, abstractmethod
+from typing import Optional, Callable, List, Dict, Any, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from .settings import Settings
+    from .schema_manager import SchemaManager
+    from .logger import Logger
+
+
+class AbstractPepeunitMqttClient(ABC):
+    """
+    Abstract base class for Pepeunit MQTT clients.
+    
+    Users can inherit from this class to create custom MQTT clients
+    that integrate with the Pepeunit ecosystem.
+    """
+    
+    def __init__(self, settings: 'Settings', schema_manager: 'SchemaManager', logger: 'Logger'):
+        self.settings = settings
+        self.schema_manager = schema_manager
+        self.logger = logger
+    
+    @abstractmethod
+    def connect(self) -> None:
+        """Connect to MQTT broker using settings configuration."""
+        pass
+    
+    @abstractmethod
+    def disconnect(self) -> None:
+        """Disconnect from MQTT broker."""
+        pass
+    
+    @abstractmethod
+    def subscribe_topics(self, topics: List[str]) -> None:
+        """
+        Subscribe to specific MQTT topics.
+        
+        Args:
+            topics: List of topic strings to subscribe to
+        """
+        pass
+    
+    @abstractmethod
+    def publish(self, topic: str, message: str) -> None:
+        """
+        Publish message to specific topic.
+        
+        Args:
+            topic: MQTT topic to publish to
+            message: Message content to publish
+        """
+        pass
+    
+    @abstractmethod
+    def set_input_handler(self, handler: Callable) -> None:
+        """
+        Set user-defined input message handler.
+        
+        Args:
+            handler: Function to handle incoming MQTT messages
+        """
+        pass
+
+
+class AbstractPepeunitRestClient(ABC):
+    """
+    Abstract base class for Pepeunit REST clients.
+    
+    Users can inherit from this class to create custom REST clients
+    that integrate with the Pepeunit API.
+    """
+    
+    def __init__(self, settings: 'Settings'):
+        self.settings = settings
+    
+    @abstractmethod
+    def download_update(self, file_path: str) -> None:
+        """
+        Download firmware update archive.
+        
+        Args:
+            unit_uuid: UUID of the unit to download update for
+            file_path: Local path where to save the downloaded file
+        """
+        pass
+    
+    @abstractmethod
+    def download_env(self, file_path: str) -> None:
+        """
+        Download env.json configuration.
+        
+        Args:
+            unit_uuid: UUID of the unit to download config for
+            file_path: Local path where to save the env.json file
+        """
+        pass
+    
+    @abstractmethod
+    def download_schema(self, file_path: str) -> None:
+        """
+        Download schema.json configuration.
+        
+        Args:
+            unit_uuid: UUID of the unit to download schema for
+            file_path: Local path where to save the schema.json file
+        """
+        pass
+    
+    @abstractmethod
+    def set_state_storage(self, state: Dict[str, Any]) -> None:
+        """
+        Store state data in Pepeunit Unit Storage.
+        
+        Args:
+            unit_uuid: UUID of the unit
+            state: State data to store
+        """
+        pass
+    
+    @abstractmethod
+    def get_state_storage(self, unit_uuid: str) -> Dict[str, Any]:
+        """
+        Retrieve state data from Pepeunit Unit Storage.
+        
+        Args:
+            unit_uuid: UUID of the unit
+            
+        Returns:
+            Retrieved state data
+        """
+        pass
+    
+    def _get_auth_headers(self) -> Dict[str, str]:
+        """
+        Get authentication headers for API requests.
+        
+        Users can override this if they need custom authentication.
+        
+        Returns:
+            Dictionary with authentication headers
+        """
+        return {
+            'accept': 'application/json',
+            'x-auth-token': self.settings.PEPEUNIT_TOKEN,
+        }
+    
+    def _get_base_url(self) -> str:
+        """
+        Get base URL for Pepeunit API.
+        
+        Users can override this if they need custom URL construction.
+        
+        Returns:
+            Base URL string
+        """
+        return f"{self.settings.HTTP_TYPE}://{self.settings.PEPEUNIT_URL}{self.settings.PEPEUNIT_APP_PREFIX}{self.settings.PEPEUNIT_API_ACTUAL_PREFIX}"

pepeunit_client/client.py

@@ -0,0 +1,308 @@
+import json
+import os
+import tempfile
+import time
+import sys
+import subprocess
+from typing import Optional, Dict, Any, Callable
+
+# Import for mocking in tests
+try:
+    import psutil
+except ImportError:
+    psutil = None
+
+from .settings import Settings
+from .file_manager import FileManager
+from .logger import Logger
+from .schema_manager import SchemaManager
+from .abstract_clients import AbstractPepeunitMqttClient, AbstractPepeunitRestClient
+from .pepeunit_mqtt_client import PepeunitMqttClient
+from .pepeunit_rest_client import PepeunitRestClient
+from .enums import BaseInputTopicType, BaseOutputTopicType, RestartMode
+
+
+class PepeunitClient:
+    def __init__(
+        self,
+        env_file_path: str,
+        schema_file_path: str,
+        log_file_path: str,
+        enable_mqtt: bool = False,
+        enable_rest: bool = False,
+        mqtt_client: Optional[AbstractPepeunitMqttClient] = None,
+        rest_client: Optional[AbstractPepeunitRestClient] = None,
+        cycle_speed: float = 0.1,
+        restart_mode: RestartMode = RestartMode.RESTART_EXEC,
+        skip_version_check=False
+    ):
+        self.env_file_path = env_file_path
+        self.schema_file_path = schema_file_path
+        self.log_file_path = log_file_path
+        self.enable_mqtt = enable_mqtt
+        self.enable_rest = enable_rest
+        self.cycle_speed = cycle_speed
+        self.restart_mode = restart_mode
+        self.skip_version_check = skip_version_check
+        
+        self.settings = Settings(env_file_path)
+        self.schema = SchemaManager(schema_file_path)
+        self.logger = Logger(log_file_path, None, self.schema, self.settings)
+
+        self.mqtt_client = (mqtt_client if mqtt_client else self._get_default_mqtt_client()) if enable_mqtt else None
+        self.rest_client = (rest_client if rest_client else self._get_default_rest_client()) if enable_rest else None
+        
+        if self.mqtt_client:
+            self.logger.mqtt_client = self.mqtt_client
+        
+        self.mqtt_input_handler: Optional[Callable] = None
+        self.mqtt_output_handler: Optional[Callable] = None
+        self.custom_update_handler: Optional[Callable] = None
+
+        self._running = False
+        self._last_state_send = 0
+        
+    def _get_default_mqtt_client(self) -> Optional[AbstractPepeunitMqttClient]:
+        return PepeunitMqttClient(self.settings, self.schema, self.logger)
+    
+    def _get_default_rest_client(self) -> Optional[AbstractPepeunitRestClient]:
+        return PepeunitRestClient(self.settings)
+    
+    def get_system_state(self) -> Dict[str, Any]:
+        if psutil is not None:
+            try:
+                memory_info = psutil.virtual_memory()
+                
+                try:
+                    cpu_freq = psutil.cpu_freq()
+                    freq = cpu_freq.current if cpu_freq else 0
+                except (AttributeError, OSError):
+                    freq = 0
+                
+                return {
+                    'millis': round(time.time() * 1000),
+                    'mem_free': memory_info.available,
+                    'mem_alloc': memory_info.total - memory_info.available,
+                    'freq': freq,
+                    'commit_version': self.settings.COMMIT_VERSION,
+                }
+            except Exception:
+                pass
+        
+        return {
+            'millis': round(time.time() * 1000),
+            'mem_free': 0,
+            'mem_alloc': 0,
+            'freq': 0,
+            'commit_version': self.settings.COMMIT_VERSION,
+        }
+    
+    def set_mqtt_input_handler(self, handler: Callable) -> None:
+
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+
+        self.mqtt_input_handler = handler
+
+        def combined_handler(msg):
+            self._base_mqtt_input_func(msg)
+            if self.mqtt_input_handler:
+                self.mqtt_input_handler(self, msg)
+
+        self.mqtt_client.set_input_handler(combined_handler)
+
+    def _base_mqtt_input_func(self, msg) -> None:
+        try:
+            for topic_key in self.schema.input_base_topic:
+                if msg.topic in self.schema.input_base_topic[topic_key]:
+                    self.logger.info(f'Get base MQTT command: {topic_key}')
+
+                    if topic_key == BaseInputTopicType.ENV_UPDATE_PEPEUNIT.value:
+                        self.download_env(self.env_file_path)
+                    elif topic_key == BaseInputTopicType.SCHEMA_UPDATE_PEPEUNIT.value:
+                        self.download_schema(self.schema_file_path)
+                    elif topic_key == BaseInputTopicType.UPDATE_PEPEUNIT.value:
+                        self._handle_update(msg)
+                    elif topic_key == BaseInputTopicType.LOG_SYNC_PEPEUNIT.value:
+                        self._handle_log_sync()
+                    break
+        except Exception as e:
+            self.logger.error(f"Error in base MQTT input handler: {str(e)}")
+    
+    def download_env(self, file_path: str) -> None:
+        if not self.rest_client:
+            raise RuntimeError("REST client is not available")
+
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+
+        self.rest_client.download_env(file_path)
+        self.settings.load_from_file()
+        self.logger.info('Success update env')
+    
+    def download_schema(self, file_path: str) -> None:
+        if not self.rest_client:
+            raise RuntimeError("REST client is not available")
+
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+
+        self.rest_client.download_schema(file_path)
+        self.schema.update_from_file()
+        self.subscribe_all_schema_topics()
+        self.logger.info('Success update schema')
+
+    def set_state_storage(self, state: str) -> None:
+        if not self.rest_client:
+            raise RuntimeError("REST client is not available")
+        
+        self.rest_client.set_state_storage(state)
+    
+    def get_state_storage(self) -> str:
+        if not self.rest_client:
+            raise RuntimeError("REST client is not available")
+        
+        return self.rest_client.get_state_storage()
+
+    def _handle_update(self, msg) -> None:
+
+        payload = json.loads(msg.payload) if msg.payload else {}
+
+        if not self.rest_client:
+            raise RuntimeError("REST client is not available")
+
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+
+        if not self.skip_version_check and self.settings.COMMIT_VERSION == payload.get('NEW_COMMIT_VERSION'):
+            self.logger.info('No update needed: current version = target version')
+            return
+            
+        if self.custom_update_handler:
+            self.custom_update_handler(self, payload)
+        else:
+            temp_dir = tempfile.gettempdir()
+            archive_path = os.path.join(temp_dir, f"update_{self.settings.unit_uuid}.tar.gz")
+            
+            self.rest_client.download_update(archive_path)
+            self.logger.info('Success download update archive', file_only=True)
+
+            self.update_device_program(archive_path)
+            self.logger.info('Success extract archive', file_only=True)
+            
+            self.logger.info("Full update cycle completed successfully")
+    
+    def update_device_program(self, archive_path: str) -> None:
+        
+        unit_directory = os.path.dirname(self.env_file_path) or os.getcwd()
+        with tempfile.TemporaryDirectory() as temp_extract_dir:
+            FileManager.extract_tar_gz(archive_path, temp_extract_dir)
+            self.logger.info(f"Extracted archive to {temp_extract_dir}")
+            
+            FileManager.copy_directory_contents(temp_extract_dir, unit_directory)
+            self.logger.info(f"Copied directory contents from {temp_extract_dir} to {unit_directory}")
+        
+        os.remove(archive_path)
+        self.logger.info(f"Archive removed {archive_path}")
+        
+        if self.restart_mode == RestartMode.RESTART_POPEN:
+            self.stop_main_cycle()
+            
+            self.logger.info('Run new main cycle in other process')
+            subprocess.Popen([sys.executable] + sys.argv)
+
+            self.logger.info('I`ll Be Back - stop this process')
+            sys.exit(0)
+        elif self.restart_mode == RestartMode.RESTART_EXEC:
+            self.stop_main_cycle()
+            
+            self.logger.info('I`ll Be Back - replacing current process')
+            os.execv(sys.executable, [sys.executable] + sys.argv)
+        elif self.restart_mode == RestartMode.ENV_SCHEMA_ONLY:
+            self.logger.info('Updating env and schema only, without restart')
+            self._update_env_schema_only()
+        elif self.restart_mode == RestartMode.NO_RESTART:
+            self.logger.info('Archive extracted, no restart or updates performed')
+    
+    def _update_env_schema_only(self) -> None:
+        self.settings.load_from_file()
+        self.schema.update_from_file()
+        
+        if self.enable_mqtt and self.mqtt_client:
+            self.subscribe_all_schema_topics()
+        
+        self.logger.info('Environment and schema updated successfully')
+
+    def _handle_log_sync(self) -> None:
+        topic = self.schema.output_base_topic[BaseOutputTopicType.LOG_PEPEUNIT.value][0]
+        log_data = self.logger.get_full_log()
+        if self.mqtt_client:
+            self.mqtt_client.publish(topic, json.dumps(log_data))
+        self.logger.info("Log sync completed")
+
+    def subscribe_all_schema_topics(self) -> None:
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+            
+        topics = []
+        
+        for topic_list in self.schema.input_base_topic.values():
+            topics.extend(topic_list)
+            
+        for topic_list in self.schema.input_topic.values():
+            topics.extend(topic_list)
+
+        self.logger.info(f'Need a subscription for {len(topic_list)} topics')
+            
+        self.mqtt_client.subscribe_topics(topics)
+
+    def publish_to_topics(self, topic_key: str, message: str) -> None:
+        if not self.mqtt_client:
+            raise RuntimeError("MQTT client is not available")
+            
+        topics = []
+        
+        if topic_key in self.schema.output_topic:
+            topics.extend(self.schema.output_topic[topic_key])
+        elif topic_key in self.schema.output_base_topic:
+            topics.extend(self.schema.output_base_topic[topic_key])
+            
+        for topic in topics:
+            self.mqtt_client.publish(topic, message)
+    
+    def _base_mqtt_output_handler(self) -> None:
+        current_time = time.time()
+        if BaseOutputTopicType.STATE_PEPEUNIT.value in self.schema.output_base_topic:
+            if current_time - self._last_state_send >= self.settings.STATE_SEND_INTERVAL:
+                topic = self.schema.output_base_topic[BaseOutputTopicType.STATE_PEPEUNIT.value][0]
+                state_data = self.get_system_state()
+                if self.mqtt_client:
+                    self.mqtt_client.publish(topic, json.dumps(state_data))
+
+                    self._last_state_send = current_time
+    
+    def run_main_cycle(self) -> None:
+        self._running = True
+        try:
+            while self._running:
+                self._base_mqtt_output_handler()
+                
+                if self.mqtt_output_handler:
+                    self.mqtt_output_handler(self)
+                
+                time.sleep(self.cycle_speed)
+                
+        except Exception as e:
+            self.logger.error(f"Error in main cycle: {str(e)}")
+        finally:
+            self._running = False
+    
+    def set_output_handler(self, output_handler: Callable) -> None:
+        self.mqtt_output_handler = output_handler
+
+    def set_custom_update_handler(self, custom_update_handler: Callable) -> None:
+        self.custom_update_handler = custom_update_handler
+
+    def stop_main_cycle(self) -> None:
+        self.logger.info(f'Main cycle stopped')
+        self._running = False

pepeunit_client/enums.py

@@ -0,0 +1,56 @@
+from enum import Enum
+
+
+class LogLevel(Enum):
+    DEBUG = 'Debug'
+    INFO = 'Info'
+    WARNING = 'Warning'
+    ERROR = 'Error'
+    CRITICAL = 'Critical'
+
+    def get_int_level(self) -> int:
+        level_mapping = {
+            LogLevel.DEBUG: 0,
+            LogLevel.INFO: 1,
+            LogLevel.WARNING: 2,
+            LogLevel.ERROR: 3,
+            LogLevel.CRITICAL: 4,
+        }
+        return level_mapping[self]
+
+
+class SearchTopicType(Enum):
+    UNIT_NODE_UUID = 'unit_node_uuid'
+    FULL_NAME = 'full_name'
+
+
+class SearchScope(Enum):
+    ALL = 'all'
+    INPUT = 'input'
+    OUTPUT = 'output'
+
+
+class DestinationTopicType(Enum):
+    INPUT_BASE_TOPIC = 'input_base_topic'
+    OUTPUT_BASE_TOPIC = 'output_base_topic'
+    INPUT_TOPIC = 'input_topic'
+    OUTPUT_TOPIC = 'output_topic'
+
+
+class BaseInputTopicType(Enum):
+    UPDATE_PEPEUNIT = 'update/pepeunit'
+    ENV_UPDATE_PEPEUNIT = 'env_update/pepeunit'
+    SCHEMA_UPDATE_PEPEUNIT = 'schema_update/pepeunit'
+    LOG_SYNC_PEPEUNIT = 'log_sync/pepeunit'
+
+
+class BaseOutputTopicType(Enum):
+    LOG_PEPEUNIT = 'log/pepeunit'
+    STATE_PEPEUNIT = 'state/pepeunit'
+
+
+class RestartMode(Enum):
+    RESTART_POPEN = 'restart_popen'
+    RESTART_EXEC = 'restart_exec'
+    ENV_SCHEMA_ONLY = 'env_schema_only'
+    NO_RESTART = 'no_restart'

pepeunit_client/file_manager.py

@@ -0,0 +1,159 @@
+import json
+import os
+import shutil
+import tarfile
+import zlib
+from typing import Any, Dict, List
+from pathlib import Path
+
+
+class FileManager:
+    @staticmethod
+    def read_json(file_path: str) -> Dict[str, Any]:
+        with open(file_path, 'r', encoding='utf-8') as f:
+            return json.load(f)
+    
+    @staticmethod
+    def write_json(file_path: str, data: Dict[str, Any], indent: int = 4) -> None:
+        directory = os.path.dirname(file_path)
+        
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        with open(file_path, 'w', encoding='utf-8') as f:
+            json.dump(data, f, indent=indent, ensure_ascii=False)
+    
+    @staticmethod
+    def copy_file(source_path: str, destination_path: str) -> None:
+        shutil.copy2(source_path, destination_path)
+    
+    @staticmethod
+    def file_exists(file_path: str) -> bool:
+        return os.path.exists(file_path)
+    
+    @staticmethod
+    def create_directory(directory_path: str) -> None:
+        os.makedirs(directory_path, exist_ok=True)
+    
+    @staticmethod
+    def extract_tar_gz(archive_path: str, extract_path: str) -> None:
+        FileManager.extract_pepeunit_archive(archive_path, extract_path)
+    
+    @staticmethod
+    def extract_pepeunit_archive(file_path: str, extract_path: str) -> None:
+        with open(file_path, 'rb') as f:
+            producer = zlib.decompressobj(wbits=9)
+            tar_data = producer.decompress(f.read()) + producer.flush()
+            tar_filepath = f'{os.path.dirname(file_path)}/temp_update.tar'
+            with open(tar_filepath, 'wb') as tar_file:
+                tar_file.write(tar_data)
+            try:
+                # Use tarfile directly with filter for security (Python 3.12+)
+                with tarfile.open(tar_filepath, 'r') as tar:
+                    # Check if filter parameter is supported
+                    try:
+                        tar.extractall(extract_path, filter='data')
+                    except TypeError:
+                        # Fallback for older Python versions
+                        tar.extractall(extract_path)
+            finally:
+                if os.path.exists(tar_filepath):
+                    os.remove(tar_filepath)
+    
+    @staticmethod
+    def copy_directory_contents(source_path: str, destination_path: str) -> None:
+        if not os.path.exists(source_path):
+            raise FileNotFoundError(f"Source directory does not exist: {source_path}")
+        
+        os.makedirs(destination_path, exist_ok=True)
+        
+        for item in os.listdir(source_path):
+            source_item = os.path.join(source_path, item)
+            destination_item = os.path.join(destination_path, item)
+            
+            if os.path.isdir(source_item):
+                shutil.copytree(source_item, destination_item, dirs_exist_ok=True)
+            else:
+                shutil.copy2(source_item, destination_item)
+    
+    @staticmethod
+    def remove_directory(directory_path: str) -> None:
+        if os.path.exists(directory_path):
+            shutil.rmtree(directory_path)
+    
+    @staticmethod
+    def append_ndjson_with_limit(file_path: str, item: Dict[str, Any], max_lines: int = None) -> None:
+        directory = os.path.dirname(file_path)
+        if directory:
+            os.makedirs(directory, exist_ok=True)
+        
+        if os.path.exists(file_path):
+            with open(file_path, 'r', encoding='utf-8') as f:
+                first_char = f.read(1)
+                if first_char == '[':
+                    try:
+                        f.seek(0)
+                        data = json.load(f)
+                        if isinstance(data, list):
+                            with open(file_path, 'w', encoding='utf-8') as fw:
+                                for it in data:
+                                    json.dump(it, fw, ensure_ascii=False)
+                                    fw.write('\n')
+                    except Exception:
+                        pass
+        
+        try:
+            with open(file_path, 'a', encoding='utf-8') as f:
+                json.dump(item, f, ensure_ascii=False)
+                f.write('\n')
+        except Exception:
+            pass
+        
+        if max_lines is not None and max_lines > 0:
+            FileManager.trim_ndjson(file_path, max_lines)
+    
+    @staticmethod
+    def iter_ndjson(file_path: str):
+        try:
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for line in f:
+                    line = line.strip()
+                    if not line:
+                        continue
+                    try:
+                        yield json.loads(line)
+                    except Exception:
+                        continue
+        except Exception:
+            return
+    
+    @staticmethod
+    def trim_ndjson(file_path: str, max_lines: int) -> None:
+        if max_lines <= 0:
+            return
+        
+        try:
+            total = 0
+            with open(file_path, 'r', encoding='utf-8') as f:
+                for _ in f:
+                    total += 1
+            
+            if total <= max_lines:
+                return
+            
+            to_skip = total - max_lines
+            tmp_path = file_path + '.tmp'
+            
+            with open(file_path, 'r', encoding='utf-8') as src, open(tmp_path, 'w', encoding='utf-8') as dst:
+                for line in src:
+                    if to_skip > 0:
+                        to_skip -= 1
+                        continue
+                    dst.write(line)
+            
+            os.replace(tmp_path, file_path)
+        except Exception:
+            if os.path.exists(tmp_path):
+                try:
+                    os.remove(tmp_path)
+                except Exception:
+                    pass