hypershell 2.6.4__py3-none-any.whl

hypershell/core/logging.py

@@ -0,0 +1,201 @@
+# SPDX-FileCopyrightText: 2025 Geoffrey Lentner
+# SPDX-License-Identifier: Apache-2.0
+
+"""Logging configuration."""
+
+
+# type annotations
+from __future__ import annotations
+from typing import Tuple, Dict, Any, Type
+
+# standard libraries
+import sys
+import uuid
+import socket
+import logging
+import functools
+import datetime
+
+# external libs
+from cmdkit.app import exit_status
+from cmdkit.config import ConfigurationError
+from cmdkit.ansi import Ansi, COLOR_STDERR
+
+# internal libs
+from hypershell.core.config import config, blame
+from hypershell.core.exceptions import write_traceback
+
+# public interface
+__all__ = ['Logger', 'HOSTNAME', 'INSTANCE', 'handler', 'initialize_logging', ]
+
+
+# Cached for later use
+HOSTNAME = socket.gethostname()
+HOSTNAME_SHORT = HOSTNAME.split('.', 1)[0]
+
+
+# Unique for every instance of hypershell
+INSTANCE = str(uuid.uuid4())
+
+
+# Canonical colors for logging messages
+level_color: Dict[str, Ansi] = {
+    'NULL': Ansi.NULL,
+    'DEVEL': Ansi.RED,
+    'TRACE': Ansi.CYAN,
+    'DEBUG': Ansi.BLUE,
+    'INFO': Ansi.GREEN,
+    'WARNING': Ansi.YELLOW,
+    'ERROR': Ansi.RED,
+    'CRITICAL': Ansi.MAGENTA
+}
+
+
+TRACE: int = logging.DEBUG - 5
+logging.addLevelName(TRACE, 'TRACE')
+
+
+DEVEL: int = 1
+logging.addLevelName(DEVEL, 'DEVEL')
+
+
+class Logger(logging.Logger):
+    """Extend Logger to implement TRACE level."""
+
+    def trace(self, msg: str, *args, **kwargs):
+        """Log 'msg % args' with severity 'TRACE'."""
+        if self.isEnabledFor(TRACE):
+            self._log(TRACE, msg, args, **kwargs)
+
+    def devel(self, msg: str, *args, **kwargs):
+        """Log 'msg % args' with severity 'DEVEL'."""
+        if self.isEnabledFor(DEVEL):
+            self._log(DEVEL, msg, args, **kwargs)
+
+    @classmethod
+    def with_name(cls: Type[Logger], name: str) -> Logger:
+        """Shorthand for `log: Logger = logging.getLogger(name)`."""
+        return logging.getLogger(name)
+
+
+# inject class back into logging library
+logging.setLoggerClass(Logger)
+
+
+def solve_relative_time(elapsed: float) -> Tuple[float, int, datetime.timedelta, str]:
+    """
+    Multiple formats of relative time since `elapsed` seconds.
+    Returns:
+        - Relative time in seconds (i.e., `elapsed`)
+        - Relative time in milliseconds
+        - Relative time as `datetime.timedelta`
+        - Relative time in dd-hh:mm:ss.sss format
+    """
+    elapsed_ms = int(elapsed * 1000)
+    reltime_delta = datetime.timedelta(seconds=elapsed)
+    reltime_delta_hours, remainder = divmod(reltime_delta.seconds, 3600)
+    reltime_delta_minutes, reltime_delta_seconds = divmod(remainder, 60)
+    reltime_delta_milliseconds = int(reltime_delta.microseconds / 1000)
+    return (
+        elapsed,
+        elapsed_ms,
+        reltime_delta,
+        f'{reltime_delta.days:02d}-{reltime_delta_hours:02d}:{reltime_delta_minutes:02d}:'
+        f'{reltime_delta_seconds:02d}.{reltime_delta_milliseconds:03d}'
+    )
+
+
+class LogRecord(logging.LogRecord):
+    """Extends LogRecord to include ANSI colors, time formats, and other attributes."""
+
+    def __init__(self, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+
+        # Context attributes
+        self.app_id = INSTANCE
+        self.hostname = HOSTNAME
+        self.hostname_short = HOSTNAME_SHORT
+        self.relative_name = self.name.split('.', 1)[-1]
+
+        # Formatting attributes
+        self.ansi_level = level_color.get(self.levelname, Ansi.NULL).value if COLOR_STDERR else ''
+        self.ansi_reset = Ansi.RESET.value if COLOR_STDERR else ''
+        self.ansi_bold = Ansi.BOLD.value if COLOR_STDERR else ''
+        self.ansi_faint = Ansi.FAINT.value if COLOR_STDERR else ''
+        self.ansi_italic = Ansi.ITALIC.value if COLOR_STDERR else ''
+        self.ansi_underline = Ansi.UNDERLINE.value if COLOR_STDERR else ''
+        self.ansi_black = Ansi.BLACK.value if COLOR_STDERR else ''
+        self.ansi_red = Ansi.RED.value if COLOR_STDERR else ''
+        self.ansi_green = Ansi.GREEN.value if COLOR_STDERR else ''
+        self.ansi_yellow = Ansi.YELLOW.value if COLOR_STDERR else ''
+        self.ansi_blue = Ansi.BLUE.value if COLOR_STDERR else ''
+        self.ansi_magenta = Ansi.MAGENTA.value if COLOR_STDERR else ''
+        self.ansi_cyan = Ansi.CYAN.value if COLOR_STDERR else ''
+        self.ansi_white = Ansi.WHITE.value if COLOR_STDERR else ''
+
+        # Timing attributes
+        (self.elapsed,
+         self.elapsed_ms,
+         self.elapsed_delta,
+         self.elapsed_hms) = solve_relative_time(self.relativeCreated / 1000)
+
+
+# inject factory back into logging library
+logging.setLogRecordFactory(LogRecord)
+
+
+class StreamHandler(logging.StreamHandler):
+    """A StreamHandler that panics on exceptions in the logging configuration."""
+
+    def handleError(self, record: LogRecord) -> None:
+        """Pretty-print message and write traceback to file."""
+        err_type, err_val, tb = sys.exc_info()
+        write_traceback(err_val, module=__name__)
+        sys.exit(exit_status.bad_config)
+
+
+def level_from_name(name: Any, source: str = 'logging.level') -> int:
+    """Get level value from `name`."""
+    label = blame(config, *source.split('.'))
+    if not isinstance(name, str):
+        raise ConfigurationError(f'Expected string for logging level, given \'{name}\' ({label})')
+    name = name.upper()
+    if name == 'TRACE':
+        return TRACE
+    elif name == 'DEVEL':
+        return DEVEL
+    elif name in ('DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'):
+        return getattr(logging, name)
+    else:
+        raise ConfigurationError(f'Unsupported logging level \'{name}\' ({label})')
+
+
+try:
+    levelname = config.logging.level
+    level = level_from_name(levelname)
+except Exception as error:
+    write_traceback(error, module=__name__)
+    sys.exit(exit_status.bad_config)
+
+
+try:
+    handler = StreamHandler(stream=sys.stderr)
+    handler.setFormatter(
+        logging.Formatter(config.logging.format,
+                          datefmt=config.logging.datefmt)
+    )
+except Exception as error:
+    write_traceback(error, module=__name__)
+    sys.exit(exit_status.bad_config)
+
+
+# null handler for library use
+logger = logging.getLogger('hypershell')
+logger.setLevel(level)
+logger.addHandler(logging.NullHandler())
+
+
+@functools.cache
+def initialize_logging() -> None:
+    """Enable logging output to the console and rotating files."""
+    logger.addHandler(handler)

hypershell/core/platform.py

@@ -0,0 +1,121 @@
+# SPDX-FileCopyrightText: 2025 Geoffrey Lentner
+# SPDX-License-Identifier: Apache-2.0
+
+"""Platform specific file paths and initialization."""
+
+
+# NOTE:
+# A lot of the work done in this core module is provided by CmdKit at this point.
+# For continuity and for fear of breaking other parts of the project we have decided
+# to leave this module in place for the time being.
+
+
+# standard libs
+import os
+import sys
+import ctypes
+import platform
+
+# external libs
+from cmdkit.config import Namespace
+from cmdkit.app import exit_status
+from cmdkit.ansi import bold, magenta
+
+# public interface
+__all__ = ['cwd', 'home', 'site', 'path', 'default_path']
+
+
+cwd = os.getcwd()
+home = os.path.expanduser('~')
+if 'HYPERSHELL_SITE' not in os.environ:
+    local_site = os.path.join(cwd, '.hypershell')
+else:
+    local_site = os.getenv('HYPERSHELL_SITE')
+    if not os.path.isdir(local_site):
+        print(f'{bold(magenta("CRITICAL"))} [{__name__}] '
+              f'Directory does not exist (HYPERSHELL_SITE={local_site})', file=sys.stderr)
+        sys.exit(exit_status.bad_config)
+
+
+if platform.system() == 'Windows':
+    is_admin = ctypes.windll.shell32.IsUserAnAdmin() == 1
+    site = Namespace(system=os.path.join(os.getenv('ProgramData'), 'HyperShell'),
+                     user=os.path.join(os.getenv('AppData'), 'HyperShell'),
+                     local=local_site)
+    path = Namespace({
+        'system': {
+            'lib': os.path.join(site.system, 'Library'),
+            'log': os.path.join(site.system, 'Logs'),
+            'config': os.path.join(site.system, 'Config.toml')},
+        'user': {
+            'lib': os.path.join(site.user, 'Library'),
+            'log': os.path.join(site.user, 'Logs'),
+            'config': os.path.join(site.user, 'Config.toml')},
+        'local': {
+            'lib': os.path.join(site.local, 'Library'),
+            'log': os.path.join(site.local, 'Logs'),
+            'config': os.path.join(site.local, 'Config.toml')}
+    })
+
+elif platform.system() == 'Darwin':
+    is_admin = os.getuid() == 0
+    site = Namespace(system='/', user=home, local=local_site)
+    path = Namespace({
+        'system': {
+            'lib': os.path.join(site['system'], 'Library', 'HyperShell'),
+            'log': os.path.join(site['system'], 'Library', 'Logs', 'HyperShell'),
+            'config': os.path.join(site['system'], 'Library', 'Preferences', 'HyperShell', 'config.toml')},
+        'user': {
+            'lib': os.path.join(site['user'], 'Library', 'HyperShell'),
+            'log': os.path.join(site['user'], 'Library', 'Logs', 'HyperShell'),
+            'config': os.path.join(site['user'], 'Library', 'Preferences', 'HyperShell', 'config.toml')},
+        'local': {
+            'lib': os.path.join(site['local'], 'Library'),
+            'log': os.path.join(site['local'], 'Logs'),
+            'config': os.path.join(site['local'], 'config.toml')}
+    })
+
+elif os.name == 'posix':  # NOTE: likely Linux
+    is_admin = os.getuid() == 0
+    site = Namespace(system='/', user=os.path.join(home, '.hypershell'),
+                     local=local_site)
+    path = Namespace({
+        'system': {
+            'lib': os.path.join(site.system, 'var', 'lib', 'hypershell'),
+            'log': os.path.join(site.system, 'var', 'log', 'hypershell'),
+            'config': os.path.join(site.system, 'etc', 'hypershell.toml')},
+        'user': {
+            'lib': os.path.join(site.user, 'lib'),
+            'log': os.path.join(site.user, 'log'),
+            'config': os.path.join(site.user, 'config.toml')},
+        'local': {
+            'lib': os.path.join(site.local, 'lib'),
+            'log': os.path.join(site.local, 'log'),
+            'config': os.path.join(site.local, 'config.toml')}
+    })
+
+else:
+    print(f'{bold(magenta("CRITICAL"))} [{__name__}] '
+          f'Platform unrecognized ({platform.system()})', file=sys.stderr)
+    sys.exit(exit_status.bad_config)
+
+
+if 'HYPERSHELL_SITE' in os.environ:
+    default_path = path.local
+else:
+    default_path = path.user if not is_admin else path.system
+
+
+# Automatically initialize default site directories
+default_dirs = [
+    default_path.lib,
+    default_path.log,
+    os.path.join(default_path.lib, 'task'),
+]
+
+
+for default_dir in default_dirs:
+    try:
+        os.makedirs(default_dir, exist_ok=True)
+    except PermissionError:
+        pass

hypershell/core/queue.py

@@ -0,0 +1,155 @@
+# SPDX-FileCopyrightText: 2025 Geoffrey Lentner
+# SPDX-License-Identifier: Apache-2.0
+
+"""Queue server/client implementation."""
+
+
+# type annotations
+from __future__ import annotations
+from typing import Dict, List, Callable, Union, Optional, Any, Iterable, Type
+from types import TracebackType
+
+# standard libs
+from multiprocessing.managers import BaseManager
+from multiprocessing import JoinableQueue
+from abc import ABC, abstractmethod
+from dataclasses import dataclass
+
+# internal libs
+from hypershell.core.config import default, config as _config
+
+# public interface
+__all__ = ['QueueConfig', 'QueueInterface', 'QueueServer', 'QueueClient']
+
+
+@dataclass
+class QueueConfig:
+    """Connection details for queue interface."""
+
+    host: str = default.server.bind
+    port: int = default.server.port
+    auth: str = default.server.auth
+    size: int = default.server.queuesize
+
+    @classmethod
+    def from_dict(cls, data: Dict[str, Union[str, int]]) -> QueueConfig:
+        """Load config from existing dictionary values."""
+        return cls(**data)
+
+    @classmethod
+    def load(cls: Type[QueueConfig]) -> QueueConfig:
+        """Initialize from global configuration."""
+        return cls.from_dict({
+            'host': _config.server.host,
+            'port': _config.server.port,
+            'auth': _config.server.auth,
+            'size': _config.server.queuesize,
+        })
+
+
+class QueueInterface(BaseManager, ABC):
+    """The queue interface provides access to three managed distributed queues."""
+
+    config: QueueConfig
+    scheduled: JoinableQueue[Optional[List[bytes]]]
+    completed: JoinableQueue[Optional[List[bytes]]]
+    heartbeat: JoinableQueue[Optional[bytes]]
+    confirmed: JoinableQueue[Optional[bytes]]
+    ready: bool = False
+
+    def __init__(self: QueueInterface, config: QueueConfig) -> None:
+        """Initialize queue interface."""
+        self.config = config
+        super().__init__(address=(self.config.host, self.config.port), authkey=self.config.auth.encode())
+
+    @classmethod
+    def new(cls: Type[QueueInterface]) -> QueueInterface:
+        """Create new interface from global configuration."""
+        return cls(config=QueueConfig.load())
+
+    @abstractmethod
+    def __enter__(self: QueueInterface) -> QueueInterface:
+        """Start server or connect from client."""
+
+    @abstractmethod
+    def __exit__(self: QueueInterface,
+                 exc_type: Optional[Type[Exception]],
+                 exc_val: Optional[Exception],
+                 exc_tb: Optional[TracebackType]) -> None:
+        """Stop or disconnect."""
+
+
+class QueueServer(QueueInterface):
+    """Server for managing queue."""
+
+    def start(self: QueueServer,
+              initializer: Optional[Callable[..., Any]] = None,
+              initargs: Iterable[Any] = ()) -> None:
+        """Initialize queues and start server."""
+        self.scheduled = JoinableQueue(maxsize=self.config.size)
+        self.completed = JoinableQueue(maxsize=self.config.size)
+        self.heartbeat = JoinableQueue(maxsize=0)
+        self.confirmed = JoinableQueue(maxsize=0)
+        self.register('_get_scheduled', callable=self._get_scheduled)
+        self.register('_get_completed', callable=self._get_completed)
+        self.register('_get_heartbeat', callable=self._get_heartbeat)
+        self.register('_get_confirmed', callable=self._get_confirmed)
+        super().start()
+        self.ready = True
+
+    def _get_scheduled(self: QueueServer) -> JoinableQueue[Optional[List[bytes]]]:
+        return self.scheduled
+
+    def _get_completed(self: QueueServer) -> JoinableQueue[Optional[List[bytes]]]:
+        return self.completed
+
+    def _get_heartbeat(self: QueueServer) -> JoinableQueue[Optional[bytes]]:
+        return self.heartbeat
+
+    def _get_confirmed(self: QueueServer) -> JoinableQueue[Optional[bytes]]:
+        return self.confirmed
+
+    def __enter__(self: QueueServer) -> QueueServer:
+        """Start the server."""
+        self.start()
+        return self
+
+    def __exit__(self: QueueServer,
+                 exc_type: Optional[Type[Exception]],
+                 exc_val: Optional[Exception],
+                 exc_tb: Optional[TracebackType]) -> None:
+        """Shutdown the server."""
+        self.shutdown()
+
+
+class QueueClient(QueueInterface):
+    """Client connection to queue manager."""
+
+    _get_scheduled: Callable[[], JoinableQueue[Optional[List[bytes]]]]
+    _get_completed: Callable[[], JoinableQueue[Optional[List[bytes]]]]
+    _get_heartbeat: Callable[[], JoinableQueue[Optional[bytes]]]
+    _get_confirmed: Callable[[], JoinableQueue[Optional[bytes]]]
+
+    def connect(self) -> None:
+        """Connect to server."""
+        self.register('_get_scheduled')
+        self.register('_get_completed')
+        self.register('_get_heartbeat')
+        self.register('_get_confirmed')
+        super().connect()
+        self.scheduled = self._get_scheduled()
+        self.completed = self._get_completed()
+        self.heartbeat = self._get_heartbeat()
+        self.confirmed = self._get_confirmed()
+        self.ready = True
+
+    def __enter__(self: QueueClient) -> QueueClient:
+        """Connect to server."""
+        self.connect()
+        return self
+
+    def __exit__(self: QueueClient,
+                 exc_type: Optional[Type[Exception]],
+                 exc_val: Optional[Exception],
+                 exc_tb: Optional[TracebackType]) -> None:
+        """Disconnect from server."""

hypershell/core/remote.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: 2025 Geoffrey Lentner
+# SPDX-License-Identifier: Apache-2.0
+
+"""Manage remote connections and data."""
+
+
+# type annotations
+from __future__ import annotations
+from typing import Tuple, Optional, Type, Union, IO
+from types import TracebackType
+
+# standard libs
+import os
+import sys
+from dataclasses import dataclass
+
+# external libs
+from paramiko import SSHClient, SFTPClient, ProxyCommand, AutoAddPolicy, SSHConfig as SSHConfigParser
+from paramiko.channel import ChannelStdinFile, ChannelFile, ChannelStderrFile
+
+# internal libs
+from hypershell.core.logging import Logger
+from hypershell.core.config import config
+from hypershell.core.thread import Thread
+
+# public interface
+__all__ = ['SSHConfig', 'SSHConnection', 'RemoteProcess', ]
+
+# initialize logger
+log = Logger.with_name(__name__)
+
+
+# Default file path to ssh configuration
+DEFAULT_SSH_CONFIG = config.ssh.config
+
+
+@dataclass
+class SSHConfig:
+    """Connection details for an SSHConnection."""
+
+    hostname: str
+    timeout: Optional[float] = None
+    username: Optional[str] = None
+    password: Optional[str] = None
+    key_filename: Optional[str] = None
+    sock: Optional[str] = None  # proxy-command
+
+    @staticmethod
+    def check_config(hostname: str, filepath: str = DEFAULT_SSH_CONFIG) -> Optional[dict]:
+        """Check to see if `hostname` is defined in `filepath`, return `paramiko.SSHConfig`."""
+        if not os.path.exists(filepath):
+            return None
+        with open(filepath, mode='r') as stream:
+            ssh_config = SSHConfigParser()
+            ssh_config.parse(stream)
+            return ssh_config.lookup(hostname)
+
+    @classmethod
+    def from_config(cls: Type[SSHConfig], hostname: str, filepath: str = DEFAULT_SSH_CONFIG) -> SSHConfig:
+        """Read configuration from file."""
+        if profile := cls.check_config(hostname, filepath):
+            return cls(**{
+                'hostname': profile.get('hostname', hostname),
+                'username': profile.get('user', None),
+                'key_filename': profile.get('identityfile', None),
+                'sock': None if 'proxycommand' not in profile else ProxyCommand(profile['proxycommand']),
+            })
+        else:
+            return cls(hostname=hostname)
+
+
+class SSHConnection:
+    """Connect to remote machine over SSH protocol."""
+
+    config: SSHConfig
+    client: SSHClient = None
+
+    sftp: Optional[SFTPClient] = None
+
+    def __init__(self: SSHConnection, hostname_or_config: Union[str, SSHConfig]) -> None:
+        """Initialize with hostname or prepared SSHConfig."""
+        if isinstance(hostname_or_config, SSHConfig):
+            self.config = hostname_or_config
+        else:
+            self.config = SSHConfig.from_config(hostname_or_config)
+
+    def __enter__(self: SSHConnection) -> SSHConnection:
+        """Open connection."""
+        self.open()
+        return self
+
+    def __exit__(self: SSHConnection,
+                 exc_type: Optional[Type[Exception]],
+                 exc_val: Optional[Exception],
+                 exc_tb: Optional[TracebackType]) -> None:
+        """Automatically close connection."""
+        self.close()
+
+    def open(self: SSHConnection) -> None:
+        """Open connection."""
+        log.debug(f'Starting SSH ({self.config.hostname})')
+        self.client = SSHClient()
+        self.client.set_missing_host_key_policy(AutoAddPolicy())
+        self.client.connect(**vars(self.config))
+
+    def close(self: SSHConnection) -> None:
+        """Close connection."""
+        if self.client:
+            log.debug('Stopping SSH')
+            self.client.close()
+        if self.sftp:
+            log.debug('Stopping SFTP')
+            self.sftp.close()
+
+    def run(self: SSHConnection, *args, **kwargs) -> Tuple[ChannelStdinFile, ChannelFile, ChannelStderrFile]:
+        """Run remote command and return <stdin>, <stdout> and <stderr>."""
+        return self.client.exec_command(*args, **kwargs)
+
+    def open_sftp(self: SSHConnection) -> None:
+        """Establish SFTP connection."""
+        if not self.sftp:
+            log.debug('Starting SFTP')
+            self.sftp = self.client.open_sftp()
+
+    def get_file(self: SSHConnection, remote_path: str, local_path: str) -> None:
+        """Use SFTP to copy remote file to local file."""
+        self.open_sftp()
+        log.trace(f'GET {self.config.hostname}:{remote_path} -> {local_path}')
+        self.sftp.get(remote_path, local_path)
+
+    def put_file(self: SSHConnection, local_path: str, remote_path: str) -> None:
+        """Use SFTP to copy local file to remote file."""
+        self.open_sftp()
+        log.trace(f'PUT {local_path} -> {self.config.hostname}:{remote_path}')
+        self.sftp.put(local_path, remote_path, confirm=True)
+
+
+class RemoteProcess(Thread):
+    """Run a command remotely over SSH and connect local file descriptors."""
+
+    conn: SSHConnection
+    command: str
+    options: dict
+
+    stdout: IO
+    stderr: IO
+
+    def __init__(self: RemoteProcess,
+                 host_or_config: Union[str, SSHConfig],
+                 command: str, stdout: IO = None, stderr: IO = None, **options) -> None:
+        """Initialize with connection and command details."""
+        super().__init__('hypershell-remote-command')
+        self.conn = SSHConnection(host_or_config)
+        self.command = command
+        self.options = options
+        self.stdout = stdout or sys.stdout
+        self.stderr = stderr or sys.stderr
+
+    def run_with_exceptions(self: RemoteProcess) -> None:
+        """Continuously redirect output."""
+        with self.conn as conn:
+            stdin, stdout, stderr = conn.run(self.command, **self.options)
+            stdin.close()
+            stdout_thread = BufferedTransport.new(stdout, self.stdout)
+            stderr_thread = BufferedTransport.new(stderr, self.stderr)
+            stdout_thread.join()
+            stderr_thread.join()
+
+    @classmethod
+    def run_and_wait(cls: Type[RemoteProcess], *args, **kwargs) -> None:
+        """Run command in blocking mode."""
+        thread = cls.new(*args, **kwargs)
+        thread.join()
+
+
+class BufferedTransport(Thread):
+    """Continuously copy data from input channel to local file stream."""
+
+    remote: ChannelFile
+    local: IO
+
+    def __init__(self: BufferedTransport, remote: ChannelFile, local: IO) -> None:
+        """Initialize with local and remote file descriptors."""
+        super().__init__('hypershell-buffered-transport')
+        self.remote = remote
+        self.local = local
+
+    def run_with_exceptions(self: BufferedTransport) -> None:
+        """Start reading data."""
+        while not self.remote.channel.exit_status_ready():
+            buffer = self.remote.readline()
+            self.local.write(buffer)