mmrelay 1.2.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mmrelay/__init__.py +5 -0
- mmrelay/__main__.py +29 -0
- mmrelay/cli.py +2013 -0
- mmrelay/cli_utils.py +746 -0
- mmrelay/config.py +956 -0
- mmrelay/constants/__init__.py +65 -0
- mmrelay/constants/app.py +29 -0
- mmrelay/constants/config.py +78 -0
- mmrelay/constants/database.py +22 -0
- mmrelay/constants/formats.py +20 -0
- mmrelay/constants/messages.py +45 -0
- mmrelay/constants/network.py +45 -0
- mmrelay/constants/plugins.py +42 -0
- mmrelay/constants/queue.py +20 -0
- mmrelay/db_runtime.py +269 -0
- mmrelay/db_utils.py +1017 -0
- mmrelay/e2ee_utils.py +400 -0
- mmrelay/log_utils.py +274 -0
- mmrelay/main.py +439 -0
- mmrelay/matrix_utils.py +3091 -0
- mmrelay/meshtastic_utils.py +1245 -0
- mmrelay/message_queue.py +647 -0
- mmrelay/plugin_loader.py +1933 -0
- mmrelay/plugins/__init__.py +3 -0
- mmrelay/plugins/base_plugin.py +638 -0
- mmrelay/plugins/debug_plugin.py +30 -0
- mmrelay/plugins/drop_plugin.py +127 -0
- mmrelay/plugins/health_plugin.py +64 -0
- mmrelay/plugins/help_plugin.py +79 -0
- mmrelay/plugins/map_plugin.py +353 -0
- mmrelay/plugins/mesh_relay_plugin.py +222 -0
- mmrelay/plugins/nodes_plugin.py +92 -0
- mmrelay/plugins/ping_plugin.py +128 -0
- mmrelay/plugins/telemetry_plugin.py +179 -0
- mmrelay/plugins/weather_plugin.py +312 -0
- mmrelay/runtime_utils.py +35 -0
- mmrelay/setup_utils.py +828 -0
- mmrelay/tools/__init__.py +27 -0
- mmrelay/tools/mmrelay.service +19 -0
- mmrelay/tools/sample-docker-compose-prebuilt.yaml +30 -0
- mmrelay/tools/sample-docker-compose.yaml +30 -0
- mmrelay/tools/sample.env +10 -0
- mmrelay/tools/sample_config.yaml +120 -0
- mmrelay/windows_utils.py +346 -0
- mmrelay-1.2.6.dist-info/METADATA +145 -0
- mmrelay-1.2.6.dist-info/RECORD +50 -0
- mmrelay-1.2.6.dist-info/WHEEL +5 -0
- mmrelay-1.2.6.dist-info/entry_points.txt +2 -0
- mmrelay-1.2.6.dist-info/licenses/LICENSE +675 -0
- mmrelay-1.2.6.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Constants package for MMRelay.
|
|
3
|
+
|
|
4
|
+
This package organizes all application constants by functional area:
|
|
5
|
+
- app: Application metadata and version information
|
|
6
|
+
- queue: Message queue configuration constants
|
|
7
|
+
- network: Network connection and timeout constants
|
|
8
|
+
- formats: Message format templates and prefixes
|
|
9
|
+
- messages: User-facing strings and templates
|
|
10
|
+
- database: Database-related constants
|
|
11
|
+
- config: Configuration section and key constants
|
|
12
|
+
- plugins: Plugin system security and validation constants
|
|
13
|
+
|
|
14
|
+
Usage:
|
|
15
|
+
from mmrelay.constants import queue
|
|
16
|
+
from mmrelay.constants.app import APP_NAME
|
|
17
|
+
from mmrelay.constants.queue import DEFAULT_MESSAGE_DELAY
|
|
18
|
+
from mmrelay.constants.plugins import DEFAULT_ALLOWED_COMMUNITY_HOSTS
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
# Re-export commonly used constants for convenience
|
|
22
|
+
from .app import APP_AUTHOR, APP_NAME
|
|
23
|
+
from .config import (
|
|
24
|
+
CONFIG_KEY_LEVEL,
|
|
25
|
+
CONFIG_SECTION_LOGGING,
|
|
26
|
+
CONFIG_SECTION_MATRIX,
|
|
27
|
+
CONFIG_SECTION_MESHTASTIC,
|
|
28
|
+
DEFAULT_LOG_LEVEL,
|
|
29
|
+
)
|
|
30
|
+
from .formats import DEFAULT_MATRIX_PREFIX, DEFAULT_MESHTASTIC_PREFIX
|
|
31
|
+
from .plugins import (
|
|
32
|
+
DEFAULT_ALLOWED_COMMUNITY_HOSTS,
|
|
33
|
+
PIP_SOURCE_FLAGS,
|
|
34
|
+
RISKY_REQUIREMENT_PREFIXES,
|
|
35
|
+
)
|
|
36
|
+
from .queue import (
|
|
37
|
+
DEFAULT_MESSAGE_DELAY,
|
|
38
|
+
MAX_QUEUE_SIZE,
|
|
39
|
+
QUEUE_HIGH_WATER_MARK,
|
|
40
|
+
QUEUE_MEDIUM_WATER_MARK,
|
|
41
|
+
)
|
|
42
|
+
|
|
43
|
+
__all__ = [
|
|
44
|
+
# App constants
|
|
45
|
+
"APP_NAME",
|
|
46
|
+
"APP_AUTHOR",
|
|
47
|
+
# Config constants
|
|
48
|
+
"CONFIG_SECTION_MATRIX",
|
|
49
|
+
"CONFIG_SECTION_MESHTASTIC",
|
|
50
|
+
"CONFIG_SECTION_LOGGING",
|
|
51
|
+
"CONFIG_KEY_LEVEL",
|
|
52
|
+
"DEFAULT_LOG_LEVEL",
|
|
53
|
+
# Queue constants
|
|
54
|
+
"DEFAULT_MESSAGE_DELAY",
|
|
55
|
+
"MAX_QUEUE_SIZE",
|
|
56
|
+
"QUEUE_HIGH_WATER_MARK",
|
|
57
|
+
"QUEUE_MEDIUM_WATER_MARK",
|
|
58
|
+
# Format constants
|
|
59
|
+
"DEFAULT_MESHTASTIC_PREFIX",
|
|
60
|
+
"DEFAULT_MATRIX_PREFIX",
|
|
61
|
+
# Plugin constants
|
|
62
|
+
"DEFAULT_ALLOWED_COMMUNITY_HOSTS",
|
|
63
|
+
"PIP_SOURCE_FLAGS",
|
|
64
|
+
"RISKY_REQUIREMENT_PREFIXES",
|
|
65
|
+
]
|
mmrelay/constants/app.py
ADDED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Application metadata constants.
|
|
3
|
+
|
|
4
|
+
Contains version information, application name, and other metadata
|
|
5
|
+
used throughout the MMRelay application.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Application identification
|
|
9
|
+
APP_NAME = "mmrelay"
|
|
10
|
+
APP_AUTHOR = None # No author directory for platformdirs
|
|
11
|
+
|
|
12
|
+
# Application display names
|
|
13
|
+
APP_DISPLAY_NAME = "MMRelay"
|
|
14
|
+
APP_FULL_NAME = "MMRelay - Meshtastic <=> Matrix Relay"
|
|
15
|
+
|
|
16
|
+
# Matrix client identification
|
|
17
|
+
MATRIX_DEVICE_NAME = "MMRelay"
|
|
18
|
+
|
|
19
|
+
# Platform-specific constants
|
|
20
|
+
WINDOWS_PLATFORM = "win32"
|
|
21
|
+
|
|
22
|
+
# Package and installation constants
|
|
23
|
+
PACKAGE_NAME_E2E = "mmrelay[e2e]"
|
|
24
|
+
PYTHON_OLM_PACKAGE = "python-olm"
|
|
25
|
+
|
|
26
|
+
# Configuration file names
|
|
27
|
+
CREDENTIALS_FILENAME = "credentials.json"
|
|
28
|
+
CONFIG_FILENAME = "config.yaml"
|
|
29
|
+
STORE_DIRNAME = "store"
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Configuration section and key constants.
|
|
3
|
+
|
|
4
|
+
Contains configuration section names, key names, and default values
|
|
5
|
+
used throughout the configuration system.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Configuration section names
|
|
9
|
+
CONFIG_SECTION_MATRIX = "matrix"
|
|
10
|
+
CONFIG_SECTION_MATRIX_ROOMS = "matrix_rooms"
|
|
11
|
+
CONFIG_SECTION_MESHTASTIC = "meshtastic"
|
|
12
|
+
CONFIG_SECTION_LOGGING = "logging"
|
|
13
|
+
CONFIG_SECTION_DATABASE = "database"
|
|
14
|
+
CONFIG_SECTION_PLUGINS = "plugins"
|
|
15
|
+
CONFIG_SECTION_COMMUNITY_PLUGINS = "community-plugins"
|
|
16
|
+
CONFIG_SECTION_CUSTOM_PLUGINS = "custom-plugins"
|
|
17
|
+
|
|
18
|
+
# Matrix configuration keys
|
|
19
|
+
CONFIG_KEY_HOMESERVER = "homeserver"
|
|
20
|
+
CONFIG_KEY_ACCESS_TOKEN = (
|
|
21
|
+
"access_token" # nosec B105 - This is a config key name, not a hardcoded password
|
|
22
|
+
)
|
|
23
|
+
CONFIG_KEY_BOT_USER_ID = "bot_user_id"
|
|
24
|
+
CONFIG_KEY_PREFIX_ENABLED = "prefix_enabled"
|
|
25
|
+
CONFIG_KEY_PREFIX_FORMAT = "prefix_format"
|
|
26
|
+
|
|
27
|
+
# Matrix rooms configuration keys
|
|
28
|
+
CONFIG_KEY_ID = "id"
|
|
29
|
+
CONFIG_KEY_MESHTASTIC_CHANNEL = "meshtastic_channel"
|
|
30
|
+
|
|
31
|
+
# Meshtastic configuration keys (additional to network.py)
|
|
32
|
+
CONFIG_KEY_MESHNET_NAME = "meshnet_name"
|
|
33
|
+
CONFIG_KEY_MESSAGE_INTERACTIONS = "message_interactions"
|
|
34
|
+
CONFIG_KEY_REACTIONS = "reactions"
|
|
35
|
+
CONFIG_KEY_REPLIES = "replies"
|
|
36
|
+
CONFIG_KEY_BROADCAST_ENABLED = "broadcast_enabled"
|
|
37
|
+
CONFIG_KEY_DETECTION_SENSOR = "detection_sensor"
|
|
38
|
+
CONFIG_KEY_MESSAGE_DELAY = "message_delay"
|
|
39
|
+
CONFIG_KEY_HEALTH_CHECK = "health_check"
|
|
40
|
+
CONFIG_KEY_ENABLED = "enabled"
|
|
41
|
+
CONFIG_KEY_HEARTBEAT_INTERVAL = "heartbeat_interval"
|
|
42
|
+
|
|
43
|
+
# Logging configuration keys
|
|
44
|
+
CONFIG_KEY_LEVEL = "level"
|
|
45
|
+
CONFIG_KEY_LOG_TO_FILE = "log_to_file"
|
|
46
|
+
CONFIG_KEY_FILENAME = "filename"
|
|
47
|
+
CONFIG_KEY_MAX_LOG_SIZE = "max_log_size"
|
|
48
|
+
CONFIG_KEY_BACKUP_COUNT = "backup_count"
|
|
49
|
+
CONFIG_KEY_COLOR_ENABLED = "color_enabled"
|
|
50
|
+
CONFIG_KEY_DEBUG = "debug"
|
|
51
|
+
|
|
52
|
+
# Database configuration keys
|
|
53
|
+
CONFIG_KEY_PATH = "path"
|
|
54
|
+
CONFIG_KEY_MSG_MAP = "msg_map"
|
|
55
|
+
CONFIG_KEY_MSGS_TO_KEEP = "msgs_to_keep"
|
|
56
|
+
CONFIG_KEY_WIPE_ON_RESTART = "wipe_on_restart"
|
|
57
|
+
|
|
58
|
+
# Plugin configuration keys
|
|
59
|
+
CONFIG_KEY_ACTIVE = "active"
|
|
60
|
+
CONFIG_KEY_CHANNELS = "channels"
|
|
61
|
+
CONFIG_KEY_UNITS = "units"
|
|
62
|
+
CONFIG_KEY_REPOSITORY = "repository"
|
|
63
|
+
CONFIG_KEY_TAG = "tag"
|
|
64
|
+
|
|
65
|
+
# Default configuration values
|
|
66
|
+
DEFAULT_LOG_LEVEL = "info"
|
|
67
|
+
DEFAULT_WEATHER_UNITS = "metric"
|
|
68
|
+
DEFAULT_WEATHER_UNITS_IMPERIAL = "imperial"
|
|
69
|
+
DEFAULT_PREFIX_ENABLED = True
|
|
70
|
+
DEFAULT_BROADCAST_ENABLED = True
|
|
71
|
+
DEFAULT_DETECTION_SENSOR = True
|
|
72
|
+
DEFAULT_HEALTH_CHECK_ENABLED = True
|
|
73
|
+
DEFAULT_HEARTBEAT_INTERVAL = 60
|
|
74
|
+
DEFAULT_COLOR_ENABLED = True
|
|
75
|
+
DEFAULT_WIPE_ON_RESTART = False
|
|
76
|
+
|
|
77
|
+
# E2EE constants
|
|
78
|
+
E2EE_KEY_SHARING_DELAY_SECONDS = 5
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Database-related constants.
|
|
3
|
+
|
|
4
|
+
Contains default values for database configuration, message retention,
|
|
5
|
+
and data management settings.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Message retention defaults
|
|
9
|
+
DEFAULT_MSGS_TO_KEEP = 500
|
|
10
|
+
DEFAULT_MAX_DATA_ROWS_PER_NODE_BASE = 100 # Base plugin default
|
|
11
|
+
DEFAULT_MAX_DATA_ROWS_PER_NODE_MESH_RELAY = 50 # Reduced for mesh relay performance
|
|
12
|
+
|
|
13
|
+
# Progress tracking
|
|
14
|
+
PROGRESS_TOTAL_STEPS = 100
|
|
15
|
+
PROGRESS_COMPLETE = 100
|
|
16
|
+
|
|
17
|
+
# Text truncation
|
|
18
|
+
DEFAULT_TEXT_TRUNCATION_LENGTH = 50
|
|
19
|
+
|
|
20
|
+
# Distance calculations
|
|
21
|
+
DEFAULT_DISTANCE_KM_FALLBACK = 1000 # Fallback distance when calculation fails
|
|
22
|
+
DEFAULT_RADIUS_KM = 5 # Default radius for location-based filtering
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Message format constants.
|
|
3
|
+
|
|
4
|
+
Contains default message prefixes, format templates, and other
|
|
5
|
+
formatting-related constants used for message display and relay.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Default message prefix formats
|
|
9
|
+
DEFAULT_MESHTASTIC_PREFIX = "{display5}[M]: "
|
|
10
|
+
DEFAULT_MATRIX_PREFIX = "[{long}/{mesh}]: "
|
|
11
|
+
|
|
12
|
+
# Port number constants for message types
|
|
13
|
+
TEXT_MESSAGE_APP = "TEXT_MESSAGE_APP"
|
|
14
|
+
DETECTION_SENSOR_APP = "DETECTION_SENSOR_APP"
|
|
15
|
+
|
|
16
|
+
# Emoji flag value
|
|
17
|
+
EMOJI_FLAG_VALUE = 1
|
|
18
|
+
|
|
19
|
+
# Default channel
|
|
20
|
+
DEFAULT_CHANNEL = 0
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
User-facing messages and string templates.
|
|
3
|
+
|
|
4
|
+
Contains error messages, log templates, command responses, and other
|
|
5
|
+
strings that are displayed to users or logged.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Log configuration defaults
|
|
9
|
+
DEFAULT_LOG_SIZE_MB = 5
|
|
10
|
+
DEFAULT_LOG_BACKUP_COUNT = 1
|
|
11
|
+
LOG_SIZE_BYTES_MULTIPLIER = 1024 * 1024 # Convert MB to bytes
|
|
12
|
+
|
|
13
|
+
# Numeric constants for comparisons
|
|
14
|
+
PORTNUM_NUMERIC_VALUE = 1 # Numeric equivalent of TEXT_MESSAGE_APP
|
|
15
|
+
DEFAULT_CHANNEL_VALUE = 0
|
|
16
|
+
|
|
17
|
+
# Message formatting constants
|
|
18
|
+
MAX_TRUNCATION_LENGTH = 20 # Maximum characters for variable truncation
|
|
19
|
+
TRUNCATION_LOG_LIMIT = 6 # Only log first N truncations to avoid spam
|
|
20
|
+
DEFAULT_MESSAGE_TRUNCATE_BYTES = 227 # Default message truncation size
|
|
21
|
+
MESHNET_NAME_ABBREVIATION_LENGTH = 4 # Characters for short meshnet names
|
|
22
|
+
SHORTNAME_FALLBACK_LENGTH = 3 # Characters for shortname fallback
|
|
23
|
+
MESSAGE_PREVIEW_LENGTH = 40 # Characters for message preview in logs
|
|
24
|
+
DISPLAY_NAME_DEFAULT_LENGTH = 5 # Default display name truncation
|
|
25
|
+
|
|
26
|
+
# Component logger names
|
|
27
|
+
COMPONENT_LOGGERS = {
|
|
28
|
+
"matrix_nio": ["nio", "nio.client", "nio.http", "nio.crypto", "nio.responses"],
|
|
29
|
+
"bleak": ["bleak", "bleak.backends"],
|
|
30
|
+
"meshtastic": [
|
|
31
|
+
"meshtastic",
|
|
32
|
+
"meshtastic.serial_interface",
|
|
33
|
+
"meshtastic.tcp_interface",
|
|
34
|
+
"meshtastic.ble_interface",
|
|
35
|
+
],
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
# Log level styling
|
|
39
|
+
LOG_LEVEL_STYLES = {
|
|
40
|
+
"DEBUG": {"color": "cyan"},
|
|
41
|
+
"INFO": {"color": "green"},
|
|
42
|
+
"WARNING": {"color": "yellow"},
|
|
43
|
+
"ERROR": {"color": "red"},
|
|
44
|
+
"CRITICAL": {"color": "red", "bold": True},
|
|
45
|
+
}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Network and connection constants.
|
|
3
|
+
|
|
4
|
+
Contains timeout values, retry limits, connection types, and other
|
|
5
|
+
network-related configuration constants.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Connection types
|
|
9
|
+
CONNECTION_TYPE_TCP = "tcp"
|
|
10
|
+
CONNECTION_TYPE_SERIAL = "serial"
|
|
11
|
+
CONNECTION_TYPE_BLE = "ble"
|
|
12
|
+
CONNECTION_TYPE_NETWORK = (
|
|
13
|
+
"network" # DEPRECATED: Legacy alias for tcp, use CONNECTION_TYPE_TCP instead
|
|
14
|
+
)
|
|
15
|
+
|
|
16
|
+
# Configuration keys for connection settings
|
|
17
|
+
CONFIG_KEY_BLE_ADDRESS = "ble_address"
|
|
18
|
+
CONFIG_KEY_SERIAL_PORT = "serial_port"
|
|
19
|
+
CONFIG_KEY_HOST = "host"
|
|
20
|
+
CONFIG_KEY_CONNECTION_TYPE = "connection_type"
|
|
21
|
+
|
|
22
|
+
# Connection retry and timing
|
|
23
|
+
DEFAULT_BACKOFF_TIME = 10 # seconds
|
|
24
|
+
DEFAULT_RETRY_ATTEMPTS = 1
|
|
25
|
+
INFINITE_RETRIES = 0 # 0 means infinite retries
|
|
26
|
+
MINIMUM_MESSAGE_DELAY = 2.0 # Minimum delay for message queue fallback
|
|
27
|
+
RECOMMENDED_MINIMUM_DELAY = (
|
|
28
|
+
2.1 # Recommended minimum delay (MINIMUM_MESSAGE_DELAY + 0.1)
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
# Matrix client timeouts
|
|
32
|
+
MATRIX_EARLY_SYNC_TIMEOUT = 2000 # milliseconds
|
|
33
|
+
MATRIX_MAIN_SYNC_TIMEOUT = 5000 # milliseconds
|
|
34
|
+
MATRIX_ROOM_SEND_TIMEOUT = 10.0 # seconds
|
|
35
|
+
MATRIX_LOGIN_TIMEOUT = 30.0 # seconds
|
|
36
|
+
MATRIX_SYNC_OPERATION_TIMEOUT = 60.0 # seconds
|
|
37
|
+
|
|
38
|
+
# Error codes
|
|
39
|
+
ERRNO_BAD_FILE_DESCRIPTOR = 9
|
|
40
|
+
|
|
41
|
+
# System detection
|
|
42
|
+
SYSTEMD_INIT_SYSTEM = "systemd"
|
|
43
|
+
|
|
44
|
+
# Time conversion
|
|
45
|
+
MILLISECONDS_PER_SECOND = 1000
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Plugin system constants.
|
|
3
|
+
|
|
4
|
+
This module contains constants related to plugin security, validation,
|
|
5
|
+
and configuration. These constants help ensure safe plugin loading and
|
|
6
|
+
execution by defining trusted sources and dangerous patterns.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from typing import Tuple
|
|
10
|
+
|
|
11
|
+
# Trusted git hosting platforms for community plugins
|
|
12
|
+
# These hosts are considered safe for plugin source repositories
|
|
13
|
+
DEFAULT_ALLOWED_COMMUNITY_HOSTS: Tuple[str, ...] = (
|
|
14
|
+
"github.com",
|
|
15
|
+
"gitlab.com",
|
|
16
|
+
"codeberg.org",
|
|
17
|
+
"bitbucket.org",
|
|
18
|
+
)
|
|
19
|
+
|
|
20
|
+
# Requirement prefixes that may indicate security risks
|
|
21
|
+
# These prefixes allow VCS URLs or direct URLs that could bypass package verification
|
|
22
|
+
RISKY_REQUIREMENT_PREFIXES: Tuple[str, ...] = (
|
|
23
|
+
"git+",
|
|
24
|
+
"ssh://",
|
|
25
|
+
"git://",
|
|
26
|
+
"hg+",
|
|
27
|
+
"bzr+",
|
|
28
|
+
"svn+",
|
|
29
|
+
"http://",
|
|
30
|
+
"https://",
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
# Pip source flags that can be followed by URLs
|
|
34
|
+
PIP_SOURCE_FLAGS: Tuple[str, ...] = (
|
|
35
|
+
"-e",
|
|
36
|
+
"--editable",
|
|
37
|
+
"-f",
|
|
38
|
+
"--find-links",
|
|
39
|
+
"-i",
|
|
40
|
+
"--index-url",
|
|
41
|
+
"--extra-index-url",
|
|
42
|
+
)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Message queue constants.
|
|
3
|
+
|
|
4
|
+
Contains configuration values for the message queue system including
|
|
5
|
+
delays, size limits, and water marks for queue management.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
# Message timing constants
|
|
9
|
+
DEFAULT_MESSAGE_DELAY = (
|
|
10
|
+
2.5 # Set above the 2.0s firmware limit to prevent message dropping
|
|
11
|
+
)
|
|
12
|
+
MINIMUM_MESSAGE_DELAY = 2.1 # Minimum delay enforced to stay above firmware limit
|
|
13
|
+
|
|
14
|
+
# Queue size management
|
|
15
|
+
MAX_QUEUE_SIZE = 500
|
|
16
|
+
QUEUE_HIGH_WATER_MARK = int(MAX_QUEUE_SIZE * 0.75) # 75% of MAX_QUEUE_SIZE
|
|
17
|
+
QUEUE_MEDIUM_WATER_MARK = int(MAX_QUEUE_SIZE * 0.50) # 50% of MAX_QUEUE_SIZE
|
|
18
|
+
|
|
19
|
+
# Queue logging thresholds
|
|
20
|
+
QUEUE_LOG_THRESHOLD = 2 # Only log queue status when size >= this value
|
mmrelay/db_runtime.py
ADDED
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Runtime utilities for managing SQLite connections used by MMRelay.
|
|
3
|
+
|
|
4
|
+
Provides a DatabaseManager that centralizes connection creation,
|
|
5
|
+
applies consistent pragmas, and exposes both synchronous context
|
|
6
|
+
managers and async helpers for executing read/write operations.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import asyncio
|
|
12
|
+
import re
|
|
13
|
+
import sqlite3
|
|
14
|
+
import threading
|
|
15
|
+
from collections.abc import Awaitable, Callable
|
|
16
|
+
from contextlib import contextmanager
|
|
17
|
+
from functools import partial
|
|
18
|
+
from typing import Any, Optional
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class DatabaseManager:
|
|
22
|
+
"""
|
|
23
|
+
Manage SQLite connections with shared pragmas and helper execution APIs.
|
|
24
|
+
|
|
25
|
+
A separate connection is maintained per thread via thread-local storage
|
|
26
|
+
(created with `check_same_thread=False`). Write operations are serialized
|
|
27
|
+
via an RLock to ensure only one writer executes at a time. Connections are
|
|
28
|
+
tracked so they can be closed when the manager is reset.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(
|
|
32
|
+
self,
|
|
33
|
+
path: str,
|
|
34
|
+
*,
|
|
35
|
+
enable_wal: bool = True,
|
|
36
|
+
busy_timeout_ms: int = 5000,
|
|
37
|
+
extra_pragmas: Optional[dict[str, Any]] = None,
|
|
38
|
+
) -> None:
|
|
39
|
+
"""
|
|
40
|
+
Create a DatabaseManager configured for the given SQLite file path.
|
|
41
|
+
|
|
42
|
+
Parameters:
|
|
43
|
+
path (str): Filesystem path to the SQLite database file.
|
|
44
|
+
enable_wal (bool): If true, connections will be configured to use Write-Ahead Logging (WAL) mode.
|
|
45
|
+
busy_timeout_ms (int): Milliseconds to wait for the database when it is busy before raising an error.
|
|
46
|
+
extra_pragmas (Optional[dict[str, Any]]): Additional PRAGMA directives to apply to each connection.
|
|
47
|
+
Keys are pragma names and values are either numeric or string pragma values. Invalid pragma
|
|
48
|
+
names or values will raise when a connection is created.
|
|
49
|
+
"""
|
|
50
|
+
self._path = path
|
|
51
|
+
self._enable_wal = enable_wal
|
|
52
|
+
self._busy_timeout_ms = busy_timeout_ms
|
|
53
|
+
self._extra_pragmas = extra_pragmas or {}
|
|
54
|
+
|
|
55
|
+
self._thread_local = threading.local()
|
|
56
|
+
self._write_lock = threading.RLock()
|
|
57
|
+
self._connections: set[sqlite3.Connection] = set()
|
|
58
|
+
self._connections_lock = threading.Lock()
|
|
59
|
+
|
|
60
|
+
# ------------------------------------------------------------------ #
|
|
61
|
+
# Internal helpers
|
|
62
|
+
# ------------------------------------------------------------------ #
|
|
63
|
+
|
|
64
|
+
def _create_connection(self) -> sqlite3.Connection:
|
|
65
|
+
"""
|
|
66
|
+
Create and configure a new sqlite3.Connection for this manager, apply configured PRAGMA directives, and register the connection for later cleanup.
|
|
67
|
+
|
|
68
|
+
Returns:
|
|
69
|
+
sqlite3.Connection: A connection configured with the manager's pragmas and tracked by the manager.
|
|
70
|
+
|
|
71
|
+
Raises:
|
|
72
|
+
sqlite3.Error: If an SQLite error occurs during connection creation or PRAGMA setup (the partially configured connection is closed before the error is propagated).
|
|
73
|
+
ValueError: If an extra PRAGMA name or string value fails validation.
|
|
74
|
+
TypeError: If an extra PRAGMA value has an unsupported type.
|
|
75
|
+
"""
|
|
76
|
+
conn = sqlite3.connect(self._path, check_same_thread=False)
|
|
77
|
+
try:
|
|
78
|
+
# Serialize PRAGMA setup to avoid concurrent WAL initialization races
|
|
79
|
+
with self._write_lock:
|
|
80
|
+
if self._busy_timeout_ms:
|
|
81
|
+
conn.execute(f"PRAGMA busy_timeout = {int(self._busy_timeout_ms)}")
|
|
82
|
+
if self._enable_wal:
|
|
83
|
+
# journal_mode pragma returns the applied mode; ignore result
|
|
84
|
+
conn.execute("PRAGMA journal_mode=WAL")
|
|
85
|
+
conn.execute("PRAGMA foreign_keys=ON")
|
|
86
|
+
for pragma, value in self._extra_pragmas.items():
|
|
87
|
+
# Validate pragma name to prevent injection.
|
|
88
|
+
if not re.fullmatch(r"[a-zA-Z_][a-zA-Z0-9_]*", pragma):
|
|
89
|
+
raise ValueError(f"Invalid pragma name provided: {pragma}")
|
|
90
|
+
# Validate and sanitize value to prevent injection
|
|
91
|
+
if isinstance(value, str):
|
|
92
|
+
# Security: Restrict pragma string values to safe characters only.
|
|
93
|
+
# This regex allows alphanumeric, underscore, hyphen, space, comma, period, and backslash.
|
|
94
|
+
# We deliberately exclude forward slash and colon to prevent path injection attacks.
|
|
95
|
+
# Backslash is allowed but trailing backslashes are blocked to prevent escape sequences.
|
|
96
|
+
#
|
|
97
|
+
# Security assumption: Configuration sources are trusted, but we validate defensively
|
|
98
|
+
# to prevent accidental or malicious injection through compromised config files.
|
|
99
|
+
# This balances security with practical SQLite pragma value requirements.
|
|
100
|
+
if not re.fullmatch(
|
|
101
|
+
r"[a-zA-Z0-9_\-\s,.\\\\]+", value
|
|
102
|
+
) or value.endswith("\\"):
|
|
103
|
+
raise ValueError(
|
|
104
|
+
f"Invalid or unsafe pragma value provided: {value}"
|
|
105
|
+
)
|
|
106
|
+
conn.execute(f"PRAGMA {pragma} = '{value}'")
|
|
107
|
+
elif isinstance(value, bool):
|
|
108
|
+
# Convert boolean values to ON/OFF for SQLite pragmas
|
|
109
|
+
conn.execute(f"PRAGMA {pragma} = {'ON' if value else 'OFF'}")
|
|
110
|
+
elif isinstance(value, (int, float)):
|
|
111
|
+
# For numeric values, ensure they're actually numeric
|
|
112
|
+
conn.execute(f"PRAGMA {pragma} = {value}")
|
|
113
|
+
else:
|
|
114
|
+
raise TypeError(f"Invalid pragma value type: {type(value)}")
|
|
115
|
+
except (sqlite3.Error, ValueError, TypeError):
|
|
116
|
+
# Ensure partially configured connection does not leak
|
|
117
|
+
conn.close()
|
|
118
|
+
raise
|
|
119
|
+
|
|
120
|
+
with self._connections_lock:
|
|
121
|
+
self._connections.add(conn)
|
|
122
|
+
return conn
|
|
123
|
+
|
|
124
|
+
def _get_connection(self) -> sqlite3.Connection:
|
|
125
|
+
"""
|
|
126
|
+
Get the thread-local SQLite connection, creating and storing a new connection if none exists.
|
|
127
|
+
|
|
128
|
+
Returns:
|
|
129
|
+
sqlite3.Connection: The per-thread SQLite connection.
|
|
130
|
+
"""
|
|
131
|
+
conn = getattr(self._thread_local, "connection", None)
|
|
132
|
+
if conn is not None:
|
|
133
|
+
try:
|
|
134
|
+
# Using cursor() is a lightweight way to check if the connection is open.
|
|
135
|
+
conn.cursor().close()
|
|
136
|
+
except sqlite3.ProgrammingError:
|
|
137
|
+
# Connection is closed, so we'll create a new one.
|
|
138
|
+
with self._connections_lock:
|
|
139
|
+
self._connections.discard(conn)
|
|
140
|
+
conn = None
|
|
141
|
+
|
|
142
|
+
if conn is None:
|
|
143
|
+
conn = self._create_connection()
|
|
144
|
+
self._thread_local.connection = conn
|
|
145
|
+
return conn
|
|
146
|
+
|
|
147
|
+
# ------------------------------------------------------------------ #
|
|
148
|
+
# Context managers
|
|
149
|
+
# ------------------------------------------------------------------ #
|
|
150
|
+
|
|
151
|
+
@contextmanager
|
|
152
|
+
def read(self) -> sqlite3.Cursor:
|
|
153
|
+
"""
|
|
154
|
+
Provide a cursor for performing read-only database operations.
|
|
155
|
+
|
|
156
|
+
The cursor is obtained from the per-thread connection and is guaranteed to be closed when the context exits. This context does not commit or roll back any transactions; it is intended for queries that do not modify persistent state.
|
|
157
|
+
|
|
158
|
+
Returns:
|
|
159
|
+
sqlite3.Cursor: A cursor tied to the manager's per-thread connection.
|
|
160
|
+
"""
|
|
161
|
+
conn = self._get_connection()
|
|
162
|
+
cursor = conn.cursor()
|
|
163
|
+
try:
|
|
164
|
+
yield cursor
|
|
165
|
+
finally:
|
|
166
|
+
cursor.close()
|
|
167
|
+
|
|
168
|
+
@contextmanager
|
|
169
|
+
def write(self) -> sqlite3.Cursor:
|
|
170
|
+
"""
|
|
171
|
+
Provide a context manager that yields a cursor for transactional write operations.
|
|
172
|
+
|
|
173
|
+
The yielded cursor is intended for executing modifying statements. The transaction is committed when the context exits normally and rolled back if an exception is raised. Write operations are serialized across threads using the manager's write lock, and the cursor is closed on exit.
|
|
174
|
+
|
|
175
|
+
Returns:
|
|
176
|
+
cursor (sqlite3.Cursor): Cursor for executing write statements; committed on success, rolled back on exception.
|
|
177
|
+
"""
|
|
178
|
+
conn = self._get_connection()
|
|
179
|
+
cursor = conn.cursor()
|
|
180
|
+
with self._write_lock:
|
|
181
|
+
try:
|
|
182
|
+
yield cursor
|
|
183
|
+
conn.commit()
|
|
184
|
+
except Exception:
|
|
185
|
+
conn.rollback()
|
|
186
|
+
raise
|
|
187
|
+
finally:
|
|
188
|
+
cursor.close()
|
|
189
|
+
|
|
190
|
+
# ------------------------------------------------------------------ #
|
|
191
|
+
# Execution helpers
|
|
192
|
+
# ------------------------------------------------------------------ #
|
|
193
|
+
|
|
194
|
+
def run_sync(
|
|
195
|
+
self,
|
|
196
|
+
func: Callable[[sqlite3.Cursor], Any],
|
|
197
|
+
*,
|
|
198
|
+
write: bool = False,
|
|
199
|
+
) -> Any:
|
|
200
|
+
"""
|
|
201
|
+
Execute a callable with a managed SQLite cursor.
|
|
202
|
+
|
|
203
|
+
Run `func` with a cursor provided by the manager; when `write` is True, the callable is executed inside a write transaction that will be committed on success and rolled back on exception.
|
|
204
|
+
|
|
205
|
+
Parameters:
|
|
206
|
+
func (Callable[[sqlite3.Cursor], Any]): A callable that receives a `sqlite3.Cursor` and returns a result.
|
|
207
|
+
write (bool): If True, execute `func` in a transactional write context; otherwise use a read-only cursor. Defaults to False.
|
|
208
|
+
|
|
209
|
+
Returns:
|
|
210
|
+
Any: The value returned by `func`.
|
|
211
|
+
"""
|
|
212
|
+
context = self.write if write else self.read
|
|
213
|
+
with context() as cursor:
|
|
214
|
+
return func(cursor)
|
|
215
|
+
|
|
216
|
+
async def run_async(
|
|
217
|
+
self,
|
|
218
|
+
func: Callable[[sqlite3.Cursor], Any],
|
|
219
|
+
*,
|
|
220
|
+
write: bool = False,
|
|
221
|
+
loop: Optional[asyncio.AbstractEventLoop] = None,
|
|
222
|
+
) -> Any:
|
|
223
|
+
"""
|
|
224
|
+
Run a database callable in the event loop's executor and return its result.
|
|
225
|
+
|
|
226
|
+
Parameters:
|
|
227
|
+
func (Callable[[sqlite3.Cursor], Any]): Callable that will be invoked with a managed SQLite cursor.
|
|
228
|
+
write (bool, optional): If true, the callable receives a cursor from a transactional write context; otherwise a read-only context is used. Defaults to False.
|
|
229
|
+
loop (asyncio.AbstractEventLoop, optional): Event loop whose executor will run the callable. If omitted, the running event loop is used.
|
|
230
|
+
|
|
231
|
+
Returns:
|
|
232
|
+
Any: The value returned by `func` when invoked with the cursor.
|
|
233
|
+
"""
|
|
234
|
+
loop = loop or asyncio.get_running_loop()
|
|
235
|
+
executor_func = partial(self.run_sync, func, write=write)
|
|
236
|
+
return await loop.run_in_executor(None, executor_func)
|
|
237
|
+
|
|
238
|
+
# ------------------------------------------------------------------ #
|
|
239
|
+
# Lifecycle
|
|
240
|
+
# ------------------------------------------------------------------ #
|
|
241
|
+
|
|
242
|
+
def close(self) -> None:
|
|
243
|
+
"""
|
|
244
|
+
Close and clean up all tracked SQLite connections.
|
|
245
|
+
|
|
246
|
+
Removes every connection from the manager's internal registry, attempts to close each connection (suppressing sqlite3.Error), and clears the current thread's stored connection reference.
|
|
247
|
+
"""
|
|
248
|
+
with self._connections_lock:
|
|
249
|
+
connections = list(self._connections)
|
|
250
|
+
self._connections.clear()
|
|
251
|
+
# Close connections while holding the lock to prevent race conditions
|
|
252
|
+
# where new connections might be created and missed during cleanup.
|
|
253
|
+
for conn in connections:
|
|
254
|
+
try:
|
|
255
|
+
conn.close()
|
|
256
|
+
except sqlite3.Error:
|
|
257
|
+
pass
|
|
258
|
+
|
|
259
|
+
# Clear thread-local references in the current thread
|
|
260
|
+
if hasattr(self._thread_local, "connection"):
|
|
261
|
+
try:
|
|
262
|
+
del self._thread_local.connection
|
|
263
|
+
except AttributeError:
|
|
264
|
+
pass
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
# Convenience alias for type hints
|
|
268
|
+
DbCallable = Callable[[sqlite3.Cursor], Any]
|
|
269
|
+
AsyncDbCallable = Callable[[sqlite3.Cursor], Awaitable[Any]]
|