hypershell 2.5.1__tar.gz → 2.6.1__tar.gz

{hypershell-2.5.1 → hypershell-2.6.1}/PKG-INFO

@@ -1,32 +1,35 @@
 Metadata-Version: 2.1
 Name: hypershell
-Version: 2.5.1
+Version: 2.6.1
 Summary: A cross-platform, high-throughput computing utility for processing shell commands over a distributed, asynchronous queue.
-Home-page: https://hyper-shell.readthedocs.io
+Home-page: https://hypershell.org
 License: Apache-2.0
 Keywords: distributed-computing,command-line-tool,shell-scripting,high-performance-computing,high-throughput-computing
 Author: Geoffrey Lentner
 Author-email: glentner@purdue.edu
-Requires-Python: >=3.10,<4.0
+Requires-Python: >=3.9,<4.0
 Classifier: Development Status :: 5 - Production/Stable
 Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: MacOS
 Classifier: Operating System :: Microsoft :: Windows
 Classifier: Operating System :: POSIX :: Linux
 Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
 Classifier: Topic :: Utilities
-Requires-Dist: cmdkit (>=2.7.5,<3.0.0)
+Provides-Extra: postgres
+Requires-Dist: cmdkit[toml] (>=2.7.7,<3.0.0)
 Requires-Dist: paramiko (>=3.4.0,<4.0.0)
-Requires-Dist: psycopg2 (>=2.9.9,<3.0.0)
+Requires-Dist: psycopg2 (>=2.9.9,<3.0.0) ; extra == "postgres"
+Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: rich (>=13.7.1,<14.0.0)
 Requires-Dist: sqlalchemy (>=2.0.29,<3.0.0)
-Requires-Dist: toml (>=0.10.2,<0.11.0)
-Requires-Dist: tomlkit (>=0.12.4,<0.13.0)
-Project-URL: Documentation, https://hyper-shell.readthedocs.io
-Project-URL: Repository, https://github.com/glentner/hypershell
+Requires-Dist: tomlkit (>=0.13.2,<0.14.0)
+Project-URL: Documentation, https://hypershell.readthedocs.io
+Project-URL: Repository, https://github.com/hypershell/hypershell
 Description-Content-Type: text/x-rst
 
 HyperShell v2: Distributed Task Execution for HPC
@@ -36,21 +39,21 @@ HyperShell v2: Distributed Task Execution for HPC
     :target: https://www.apache.org/licenses/LICENSE-2.0
     :alt: License
 
-.. image:: https://img.shields.io/pypi/v/hypershell.svg?style=flat&color=blue
-    :target: https://pypi.org/project/hypershell
-    :alt: PyPI Version
+.. image:: https://img.shields.io/github/v/release/hypershell/hypershell?sort=semver
+    :target: https://github.com/hypershell/hypershell/releases
+    :alt: Github Release
 
-.. image:: https://img.shields.io/pypi/pyversions/hypershell.svg?logo=python&logoColor=white&style=flat
-    :target: https://pypi.org/project/hypershell
+.. image:: https://img.shields.io/badge/Python-3.10+-blue.svg
+    :target: https://www.python.org/downloads
     :alt: Python Versions
 
-.. image:: https://readthedocs.org/projects/hyper-shell/badge/?version=latest&style=flat
-    :target: https://hyper-shell.readthedocs.io
-    :alt: Documentation
+.. image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg
+    :target: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
+    :alt: Code of Conduct
 
-.. image:: https://static.pepy.tech/badge/hyper-shell
-    :target: https://pepy.tech/project/hyper-shell
-    :alt: Downloads
+.. image:: https://readthedocs.org/projects/hypershell/badge/?version=latest
+    :target: https://hypershell.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
 
 |
 
@@ -77,7 +80,7 @@ the user ergonomics we provide. Novel design elements include but are not limite
 Documentation
 -------------
 
-Documentation is available at `hyper-shell.readthedocs.io <https://hyper-shell.readthedocs.io>`_.
+Documentation is available at `hypershell.readthedocs.io <https://hypershell.readthedocs.io>`_.
 For basic usage information on the command line use: ``hs --help``. For a more
 comprehensive usage guide on the command line you can view the manual page with 
 ``man hs``.

{hypershell-2.5.1 → hypershell-2.6.1}/README.rst

@@ -5,21 +5,21 @@ HyperShell v2: Distributed Task Execution for HPC
     :target: https://www.apache.org/licenses/LICENSE-2.0
     :alt: License
 
-.. image:: https://img.shields.io/pypi/v/hypershell.svg?style=flat&color=blue
-    :target: https://pypi.org/project/hypershell
-    :alt: PyPI Version
+.. image:: https://img.shields.io/github/v/release/hypershell/hypershell?sort=semver
+    :target: https://github.com/hypershell/hypershell/releases
+    :alt: Github Release
 
-.. image:: https://img.shields.io/pypi/pyversions/hypershell.svg?logo=python&logoColor=white&style=flat
-    :target: https://pypi.org/project/hypershell
+.. image:: https://img.shields.io/badge/Python-3.10+-blue.svg
+    :target: https://www.python.org/downloads
     :alt: Python Versions
 
-.. image:: https://readthedocs.org/projects/hyper-shell/badge/?version=latest&style=flat
-    :target: https://hyper-shell.readthedocs.io
-    :alt: Documentation
+.. image:: https://img.shields.io/badge/Contributor%20Covenant-2.1-4baaaa.svg
+    :target: https://www.contributor-covenant.org/version/2/1/code_of_conduct/
+    :alt: Code of Conduct
 
-.. image:: https://static.pepy.tech/badge/hyper-shell
-    :target: https://pepy.tech/project/hyper-shell
-    :alt: Downloads
+.. image:: https://readthedocs.org/projects/hypershell/badge/?version=latest
+    :target: https://hypershell.readthedocs.io/en/latest/?badge=latest
+    :alt: Documentation Status
 
 |
 
@@ -46,7 +46,7 @@ the user ergonomics we provide. Novel design elements include but are not limite
 Documentation
 -------------
 
-Documentation is available at `hyper-shell.readthedocs.io <https://hyper-shell.readthedocs.io>`_.
+Documentation is available at `hypershell.readthedocs.io <https://hypershell.readthedocs.io>`_.
 For basic usage information on the command line use: ``hs --help``. For a more
 comprehensive usage guide on the command line you can view the manual page with 
 ``man hs``.

{hypershell-2.5.1 → hypershell-2.6.1}/pyproject.toml

@@ -1,12 +1,12 @@
 [tool.poetry]
 name = "hypershell"
-version = "2.5.1"
+version = "2.6.1"
 description = "A cross-platform, high-throughput computing utility for processing shell commands over a distributed, asynchronous queue."
 readme = "README.rst"
 license = "Apache-2.0"
-homepage = "https://hyper-shell.readthedocs.io"
-documentation = "https://hyper-shell.readthedocs.io"
-repository = "https://github.com/glentner/hypershell"
+homepage = "https://hypershell.org"
+documentation = "https://hypershell.readthedocs.io"
+repository = "https://github.com/hypershell/hypershell"
 authors = [
     "Geoffrey Lentner <glentner@purdue.edu>"
 ]
@@ -20,9 +20,11 @@ keywords = [
 classifiers = [
     "Development Status :: 5 - Production/Stable",
     "Topic :: Utilities",
+    "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
     "Operating System :: POSIX :: Linux",
     "Operating System :: MacOS",
     "Operating System :: Microsoft :: Windows",
@@ -35,15 +37,18 @@ hyper-shell = "hypershell:main"  # NOTE: do not remove this
 hs = "hypershell:main"
 
 [tool.poetry.dependencies]
-python = ">=3.10,<4.0"
-cmdkit = "^2.7.5"
-toml = "^0.10.2"
-tomlkit = "^0.12.4"
+python = ">=3.9,<4.0"
+cmdkit = {version = "^2.7.7", extras = ["toml"]}
+pyyaml = "^6.0.2"
+tomlkit = "^0.13.2"
 sqlalchemy = "^2.0.29"
 rich = "^13.7.1"
 paramiko = "^3.4.0"
 psycopg2 = {version = "^2.9.9", optional = true}
 
+[tool.poetry.extras]
+postgres = ["psycopg2"]
+
 [tool.poetry.group.docs.dependencies]
 sphinx = "^7.2.6"
 sphinx-sitemap = "^2.5.1"
@@ -59,9 +64,7 @@ furo = "^2024.1.29"
 [tool.poetry.group.dev.dependencies]
 psycopg2 = "^2.9.9"
 pytest = "^8.1.1"
-pytest-cov = "^5.0.0"
 hypothesis = "^6.100.0"
-ipython = "^8.23.0"
 sphinx-autobuild = "^2024.2.4"
 
 [build-system]

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/__init__.py

@@ -7,6 +7,7 @@
 # standard libs
 import sys
 from importlib.metadata import version as get_version
+from platform import python_version
 
 # external libs
 from cmdkit.app import Application, ApplicationGroup
@@ -91,11 +92,11 @@ class HyperShellApp(ApplicationGroup):
     """Top-level application class for console application."""
 
     interface = Interface(APP_NAME, APP_USAGE, APP_HELP)
-    interface.add_argument('-v', '--version', action='version', version=__version__)
+    interface.add_argument('-v', '--version', action='version',
+                           version=f'HyperShell v{__version__} (Python {python_version()})')
     interface.add_argument('--citation', action='version', version=__citation__)
     interface.add_argument('command')
 
-    command = None
     commands = {
         'submit': SubmitApp,
         'server': ServerApp,

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/client.py

@@ -563,8 +563,7 @@ class TaskExecutor(StateMachine):
                              stdout=self.redirect_output, stderr=self.redirect_errors,
                              cwd=config.task.cwd, env=env)
         log.info(f'Running task ({self.task.id})')
-        log.debug(f'Running task ({self.task.id}: {self.task.command})')
-        log.trace(f'Running task ({self.task.id}: pid={self.process.pid}, argv={self.task.command})')
+        log.debug(f'Running task ({self.task.id})[{self.process.pid}]: {self.task.command}')
         return TaskState.WAIT_TASK
 
     def wait_task(self: TaskExecutor) -> TaskState:

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/cluster/local.py

@@ -197,7 +197,8 @@ class LocalCluster(Thread):
         """Start child threads, wait."""
         set_client_standalone(False)
         self.server.start()
-        time.sleep(2)  # NOTE: give the server a chance to start
+        while not self.server.queue.ready:
+            time.sleep(0.1)
         self.client.start()
         self.client.join()
         self.server.join()

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/cluster/remote.py

@@ -180,7 +180,6 @@ class RemoteCluster(Thread):
 
     See Also:
         - :class:`~hypershell.server.ServerThread`
-        - :class:`~hypershell.client.ClientThread`
         - :meth:`~hypershell.cluster.remote.run_cluster`
     """
 
@@ -248,7 +247,8 @@ class RemoteCluster(Thread):
     def run_with_exceptions(self: RemoteCluster) -> None:
         """Start child threads, wait."""
         self.server.start()
-        time.sleep(2)  # NOTE: give the server a chance to start
+        while not self.server.queue.ready:
+            time.sleep(0.1)
         log.debug(f'Launching clients: {self.client_argv}')
         self.clients = Popen(self.client_argv,
                              stdout=sys.stdout, stderr=sys.stderr,
@@ -708,7 +708,8 @@ class AutoScalingCluster(Thread):
     def run_with_exceptions(self: AutoScalingCluster) -> None:
         """Start child threads, wait."""
         self.server.start()
-        time.sleep(2)  # NOTE: give the server a chance to start
+        while not self.server.queue.ready:
+            time.sleep(0.1)
         self.autoscaler.start()
         self.autoscaler.join()
         self.server.join()

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/cluster/ssh.py

@@ -56,7 +56,7 @@ def run_ssh(**options) -> None:
         ... )
 
     See Also:
-        - :class:`~hypershell.cluster.remote.SSHCluster`
+        - :class:`~hypershell.cluster.ssh.SSHCluster`
     """
     thread = SSHCluster.new(**options)
     try:
@@ -244,7 +244,8 @@ class SSHCluster(Thread):
     def run_with_exceptions(self: SSHCluster) -> None:
         """Start child threads, wait."""
         self.server.start()
-        time.sleep(2)  # NOTE: give the server a chance to start
+        while not self.server.queue.ready:
+            time.sleep(0.1)
         self.clients = []
         for argv in self.client_argv:
             log.debug(f'Launching client: {argv}')

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/config.py

@@ -17,7 +17,7 @@ import contextlib
 import subprocess
 
 # external libs
-import toml
+import tomlkit
 from pygments.styles import STYLE_MAP as CONSOLE_THEMES
 from cmdkit.app import Application, ApplicationGroup
 from cmdkit.cli import Interface, ArgumentError
@@ -239,19 +239,9 @@ class ConfigGetApp(Application):
     def format_section(self: ConfigGetApp, value: dict) -> str:
         """Format an entire section for output."""
         if self.varpath == '.':
-            value = toml.dumps(value)
+            return tomlkit.dumps(value)
         else:
-            value = toml.dumps({self.varpath: value})
-        # NOTE: Fix weird formatting of section headings.
-        #       The `toml.dumps` output has unnecessary quoting.
-        lines = []
-        for line in value.strip().split('\n'):
-            if not line.startswith('['):
-                lines.append(line)
-            else:
-                lines.append(line.replace('"', ''))
-        value = '\n'.join(lines)
-        return value
+            return tomlkit.dumps({self.varpath: value})
 
 
 SET_PROGRAM = 'hs config set'
@@ -442,10 +432,8 @@ class ConfigApp(ApplicationGroup):
     """Manage configuration."""
 
     interface = Interface(PROGRAM, USAGE, HELP)
-
     interface.add_argument('command')
 
-    command = None
     commands = {'get': ConfigGetApp,
                 'set': ConfigSetApp,
                 'edit': ConfigEditApp,

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/core/config.py

@@ -335,6 +335,8 @@ else:
 
 
 T = TypeVar('T')
+
+
 def __collapse_if_list_impl(value: Union[T, List[str]]) -> Union[T, str]:
     """If `value` is a list, collapse it to a path-like list (with ':' or ';')."""
     return value if not isinstance(value, list) else PATH_DELIMITER.join([str(member) for member in value])

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/core/fsm.py

@@ -9,10 +9,12 @@ from __future__ import annotations
 from typing import Dict, Callable, Type
 
 # standard libs
-# import time  # NOTE: commented section below
-# import random
 from enum import Enum
 from abc import ABC
+# FUZZ: import time
+# FUZZ: import random
+# PERF: from collections import defaultdict
+# PERF: from time import perf_counter
 
 # internal libs
 from hypershell.core.exceptions import write_traceback
@@ -38,6 +40,10 @@ class StateMachine(ABC):
 
     __should_halt: bool = False
 
+    # NOTE: Only needed during performance profiling
+    # PERF: __perf_counter: float = 0
+    # PERF: __perf_data: Dict[int, float]
+
     def next(self) -> State:
         """Return next state (halt if necessary)."""
         previous_state = self.state
@@ -46,21 +52,30 @@ class StateMachine(ABC):
                 return self.states.HALT  # noqa: HALT defined in implemented State enums
             else:
                 action = self.actions.get(previous_state)
+                # PERF: self.__perf_counter = perf_counter()
                 next_state = action()
+                # PERF: self.__perf_data[previous_state.value] += perf_counter() - self.__perf_counter
         except Exception as error:
-            log.critical(f'Uncaught exception from {self.__class__}')
-            write_traceback(error, logger=log, module=__name__)
+            # NOTE: Only non-RuntimeError instances are "unexpected"
+            if not isinstance(error, RuntimeError):
+                log.critical(f'Uncaught exception from {self.__class__}')
+                write_traceback(error, logger=log, module=__name__)
             raise
         else:
             # NOTE: Development aids not typically engaged
-            # time.sleep(random.uniform(0, 5))  # FUZZ
-            # log.devel(f'{self.__class__.__name__}: {previous_state} -> {next_state}')
+            # FUZZ: time.sleep(random.uniform(0, 5))  # FUZZ
+            # FUZZ: log.devel(f'{self.__class__.__name__}: {previous_state} -> {next_state}')
             return next_state
 
     def run(self) -> None:
         """Run machine until state is set to `HALT`."""
+        # PERF: self.__perf_data = defaultdict(lambda: 0)
         while self.state is not self.states.HALT:  # noqa: HALT defined in implemented State enums
             self.state = self.next()
+        # PERF: time_total = sum(self.__perf_data.values())
+        # PERF: for key, value in self.__perf_data.items():
+        # PERF:     t = 100 * value / time_total
+        # PERF:     log.trace(f'Profiler[{self.__class__.__name__}] {self.states(key).name}: {t:.3f}')
 
     def halt(self) -> None:
         """Set flag to signal for termination."""

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/core/queue.py

@@ -55,6 +55,7 @@ class QueueInterface(BaseManager, ABC):
     completed: JoinableQueue[Optional[List[bytes]]]
     heartbeat: JoinableQueue[Optional[bytes]]
     confirmed: JoinableQueue[Optional[bytes]]
+    ready: bool = False
 
     def __init__(self: QueueInterface, config: QueueConfig) -> None:
         """Initialize queue interface."""
@@ -94,6 +95,7 @@ class QueueServer(QueueInterface):
         self.register('_get_heartbeat', callable=self._get_heartbeat)
         self.register('_get_confirmed', callable=self._get_confirmed)
         super().start()
+        self.ready = True
 
     def _get_scheduled(self: QueueServer) -> JoinableQueue[Optional[List[bytes]]]:
         return self.scheduled
@@ -139,6 +141,7 @@ class QueueClient(QueueInterface):
         self.completed = self._get_completed()
         self.heartbeat = self._get_heartbeat()
         self.confirmed = self._get_confirmed()
+        self.ready = True
 
     def __enter__(self: QueueClient) -> QueueClient:
         """Connect to server."""

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/core/signal.py

@@ -10,8 +10,9 @@ from typing import Optional, Final, Dict
 from types import FrameType
 
 # standard libs
+import platform
 from signal import signal as register
-from signal import SIGUSR1, SIGUSR2, SIGINT, SIGTERM, SIGKILL
+
 
 # internal libs
 from hypershell.core.logging import Logger
@@ -22,6 +23,19 @@ __all__ = ['check_signal', 'RECEIVED', 'SIGNAL_MAP',
            'SIGUSR1', 'SIGUSR2', 'SIGINT', 'SIGTERM', 'SIGKILL']
 
 
+if platform.system() != 'Windows':
+    from signal import SIGUSR1, SIGUSR2, SIGINT, SIGTERM, SIGKILL
+else:
+    # NOTE:
+    # Windows does not provide the signal facility
+    # While valid, these stubs have no effect because on Windows we never signal
+    SIGUSR1: Final[int] = 30
+    SIGUSR2: Final[int] = 31
+    SIGINT: Final[int] = 2
+    SIGTERM: Final[int] = 15
+    SIGKILL: Final[int] = 9
+
+
 # initialize logger
 log = Logger.with_name(__name__)
 
@@ -44,14 +58,22 @@ SIGNAL_MAP: Final[Dict[int, str]] = {
 }
 
 
-def handler(signum: int, frame: Optional[FrameType]) -> None:
+def handler(signum: int, frame: Optional[FrameType]) -> None:  # noqa: unused frame
     """Generic handler assigns `signum` to global variable."""
     log.debug(f'Received signal {signum}: {SIGNAL_MAP.get(signum, "???")}')
     global RECEIVED
     RECEIVED = signum
 
 
-def register_handlers() -> None:
-    """Register signal handlers for client."""
-    register(SIGUSR1, handler)
-    register(SIGUSR2, handler)
+if platform.system() == 'Windows':
+
+    def register_handlers() -> None:
+        """Empty function does nothing on Windows."""
+        pass
+
+else:
+
+    def register_handlers() -> None:
+        """Register signal handlers for client."""
+        register(SIGUSR1, handler)
+        register(SIGUSR2, handler)

hypershell-2.6.1/src/hypershell/core/tag.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2024 Geoffrey Lentner
+# SPDX-License-Identifier: Apache-2.0
+
+"""Tag interface and parsing."""
+
+
+# type annotations
+from __future__ import annotations
+from typing import Dict, List, Optional, Type
+
+# standard libs
+from dataclasses import dataclass
+
+# internal libs
+from hypershell.core.types import JSONValue, smart_coerce
+
+# public interface
+__all__ = ['Tag', ]
+
+
+@dataclass
+class Tag:
+    """Tag specification."""
+
+    name: str
+    value: JSONValue = ''
+
+    def to_dict(self: Tag) -> Dict[str, JSONValue]:
+        """Format tag specification as dictionary."""
+        return {self.name: self.value, }
+
+    @classmethod
+    def from_cmdline(cls: Type[Tag], arg: str) -> Tag:
+        """Construct from command-line `arg`."""
+        tag_part = arg.strip().split(':', 1)
+        if len(tag_part) == 1:
+            return cls(name=tag_part[0].strip())
+        else:
+            name, value = tag_part[0].strip(), smart_coerce(tag_part[1].strip())
+            # Task.ensure_valid_tag({name: value})
+            return cls(name, value)
+
+    @classmethod
+    def parse_cmdline_list(cls: Type[Tag], args: List[str]) -> Dict[str, Optional[JSONValue]]:
+        """Parse command-line list of tags."""
+        return {tag.name: tag.value for tag in map(cls.from_cmdline, args)}
+

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/data/core.py

@@ -24,7 +24,7 @@ from sqlalchemy.exc import ArgumentError
 # internal libs
 from hypershell.core.config import config
 from hypershell.core.logging import handler
-from hypershell.core.exceptions import write_traceback
+from hypershell.core.exceptions import display_critical, write_traceback
 
 # public interface
 __all__ = ['DatabaseURL', 'engine', 'Session', 'config', 'in_memory', 'schema', ]
@@ -220,6 +220,13 @@ try:
     engine = get_engine()
     factory = sessionmaker(bind=engine)
     Session = scoped_session(factory)
+except ModuleNotFoundError as error:
+    if 'psycopg2' in error.args[0]:
+        display_critical(f'Missing optional dependency "psycopg2" needed for PostgreSQL', module=__name__)
+        sys.exit(exit_status.runtime_error)
+    else:
+        write_traceback(error, module=__name__)
+        sys.exit(exit_status.bad_config)
 except Exception as error:
     write_traceback(error, module=__name__)
     sys.exit(exit_status.bad_config)

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/data/model.py

@@ -6,7 +6,7 @@
 
 # type annotations
 from __future__ import annotations
-from typing import List, Dict, Any, Type, TypeVar, Union, Optional
+from typing import List, Dict, Tuple, Any, Type, TypeVar, Union, Optional
 
 # standard libs
 import re
@@ -20,12 +20,13 @@ from sqlalchemy.orm import Query, DeclarativeBase, Mapped, mapped_column
 from sqlalchemy.orm.exc import NoResultFound, MultipleResultsFound
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.types import Integer, DateTime, Text, Boolean, JSON as _JSON
-from sqlalchemy.dialects.postgresql import UUID as POSTGRES_UUID, JSONB as POSTGRES_JSON
+from sqlalchemy.dialects.postgresql import SMALLINT, UUID as POSTGRES_UUID, JSONB as POSTGRES_JSON
 
 # internal libs
 from hypershell.core.logging import Logger, HOSTNAME, INSTANCE
 from hypershell.core.heartbeat import Heartbeat
 from hypershell.core.types import JSONValue
+from hypershell.core.tag import Tag
 from hypershell.data.core import schema, Session
 
 # public interface
@@ -77,6 +78,7 @@ def from_json_type(value: JSONValue) -> Union[JSONValue, VT]:
 UUID = Text().with_variant(POSTGRES_UUID(as_uuid=False), 'postgresql')
 TEXT = Text()
 INTEGER = Integer()
+SMALL_INTEGER = Integer().with_variant(SMALLINT, 'postgresql')
 DATETIME = DateTime(timezone=True)
 BOOLEAN = Boolean()
 JSON = _JSON().with_variant(POSTGRES_JSON(), 'postgresql')
@@ -180,6 +182,26 @@ class Entity(DeclarativeBase):
         """Update by `id` with `changes`."""
         cls.update_all([{'id': id, **changes}, ])
 
+    @classmethod
+    def delete_all(cls: Type[Entity], items: List[Entity]) -> List[Entity]:
+        """Delete records from database."""
+        try:
+            for item in items:
+                Session.delete(item)
+            Session.commit()
+        except Exception:
+            Session.rollback()
+            raise
+        else:
+            for item in items:
+                log.trace(f'Deleted {cls.__tablename__} ({item.id})')  # noqa: id not defined on base
+            return items
+
+    @classmethod
+    def delete(cls: Type[Entity], item: Entity) -> None:
+        """Delete single item from database."""
+        cls.delete_all([item, ])
+
     @classmethod
     def from_id(cls: Type[Entity], id: str) -> Entity:
         """Load by unique `id`."""
@@ -211,12 +233,12 @@ class Task(Entity):
     command: Mapped[Optional[str]] = mapped_column(TEXT, nullable=True)
     start_time: Mapped[Optional[datetime]] = mapped_column(DATETIME, nullable=True)
     completion_time: Mapped[Optional[datetime]] = mapped_column(DATETIME, nullable=True)
-    exit_status: Mapped[Optional[int]] = mapped_column(INTEGER, nullable=True)
+    exit_status: Mapped[Optional[int]] = mapped_column(SMALL_INTEGER, nullable=True)
 
     outpath: Mapped[Optional[str]] = mapped_column(TEXT, nullable=True)
     errpath: Mapped[Optional[str]] = mapped_column(TEXT, nullable=True)
 
-    attempt: Mapped[int] = mapped_column(INTEGER, nullable=False)
+    attempt: Mapped[int] = mapped_column(SMALL_INTEGER, nullable=False)
     retried: Mapped[bool] = mapped_column(BOOLEAN, nullable=False)
 
     waited: Mapped[Optional[int]] = mapped_column(INTEGER, nullable=True)
@@ -277,30 +299,53 @@ class Task(Entity):
             tag: Dict[str, JSONValue] = None, **other) -> Task:
         """Create a new Task."""
         cls.ensure_valid_tag(tag)
+        args, inline_tags = cls.split_argline(args)
+        tag = {**(tag or {}), **inline_tags}
         return Task(id=str(gen_uuid()), args=str(args).strip(),
                     submit_id=INSTANCE, submit_host=HOSTNAME, submit_time=datetime.now().astimezone(),
-                    attempt=attempt, retried=retried, tag=(tag or {}), **other)
+                    attempt=attempt, retried=retried, tag=tag, **other)
+
+    @classmethod
+    def split_argline(cls: Type[Task], args: str) -> Tuple[str, Dict[str, JSONValue]]:
+        """Separate input args from possible inline tag comment."""
+        if match := re.search(r'#\s*HYPERSHELL:?', args):
+            try:
+                tags = Tag.parse_cmdline_list(args[match.end():].strip().split())
+                cls.ensure_valid_tag(tags)
+            except (ValueError, TypeError) as error:
+                raise RuntimeError(f'Failed to parse inline tags ({error}, from: "{args}")') from error
+            args = args[:match.start()]
+            return args, tags
+        else:
+            return args, {}
 
     @staticmethod
-    def ensure_valid_tag(tag: Dict[str, JSONValue]) -> None:
+    def ensure_valid_tag(tag: Optional[Dict[str, JSONValue]]) -> None:
         """Check tag dictionary and raise if invalid."""
-        if not isinstance(tag, (dict, type(None))):
-            raise TypeError('Expected dict or None for tag data')
         if tag is None:
             return
+        if not isinstance(tag, dict):
+            raise TypeError('Expected dict for tag data')
         for key, value in tag.items():
             if not isinstance(key, str):
                 raise TypeError(f'Tag key, {key} ({type(key)}) is not string')
+            if len(key.strip()) == 0:
+                raise ValueError(f'Tag key was empty, "{key}:{value}"')
+            if len(key.strip()) > 120:
+                raise ValueError(f'Tag key size ({len(value)}) exceeds 120 characters ({key}:{value})')
+            if not re.match(r'^[A-Za-z0-9_.+-]+$', key):
+                raise ValueError(f'Tag key must only contain alphanumerics and basic symbols [+._-]: '
+                                 f'"{key}:{value}"')
             if not isinstance(value, (str, int, float, bool, type(None))):
                 raise TypeError(f'Invalid type for tag value, {type(value)})')
             if isinstance(value, str):
                 if not value.strip():
                     return  # Empty value is a naked tag (no value).
                 if len(value) > 120:
-                    raise ValueError(f'Tag size ({len(value)}) exceeds 120 characters ({key}: {value})')
+                    raise ValueError(f'Tag value size ({len(value)}) exceeds 120 characters ({key}:{value})')
                 if not re.match(r'^[A-Za-z0-9_.+-]+$', value):
-                    raise ValueError(f'Tag must only contain alphanumerics and basic symbols [+._-]: '
-                                     f'({key}: {value})')
+                    raise ValueError(f'Tag value must only contain alphanumerics and basic symbols [+._-]: '
+                                     f'"{key}:{value}"')
 
     @classmethod
     def select_new(cls: Type[Task], limit: int) -> List[Task]:
@@ -556,7 +601,6 @@ class Task(Entity):
 # Indices for efficient queries
 index_scheduled = Index('task_scheduled_index', Task.schedule_time)
 index_retried = Index('task_retries_index', Task.exit_status, Task.retried)
-index_client_completed = Index('task_client_completed_index', Task.client_id, Task.completion_time)
 
 
 class Client(Entity):

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/server.py

@@ -92,9 +92,6 @@ class SchedulerState(State, Enum):
     HALT = 5
 
 
-# Note:
-#   Unless specified otherwise for larger problems, a bundle of size one allows
-#   for greater concurrency on smaller workloads.
 DEFAULT_BUNDLESIZE: Final[int] = default.server.bundlesize
 """Default size for task bundles."""
 

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/submit.py

@@ -53,7 +53,7 @@ from queue import Queue, Empty as QueueEmpty, Full as QueueFull
 # external libs
 from cmdkit.config import ConfigurationError
 from cmdkit.app import Application
-from cmdkit.cli import Interface
+from cmdkit.cli import Interface, ArgumentError
 
 # internal libs
 from hypershell.core.logging import Logger
@@ -760,6 +760,7 @@ class SubmitApp(Application):
     auto_initdb: bool = False
     interface.add_argument('--initdb', action='store_true', dest='auto_initdb')
 
+    tags: Dict[str, JSONValue] = {}
     taglist: List[str] = None
     interface.add_argument('--tag', nargs='*', default=[], dest='taglist')
 
@@ -777,8 +778,7 @@ class SubmitApp(Application):
     def submit_all(self: SubmitApp) -> None:
         """Submit all tasks from source."""
         self.count = submit_from(self.source, template=self.template,
-                                 bundlesize=self.bundlesize, bundlewait=self.bundlewait,
-                                 tags=Tag.parse_cmdline_list(self.taglist))
+                                 bundlesize=self.bundlesize, bundlewait=self.bundlewait, tags=self.tags)
 
     @staticmethod
     def check_config():
@@ -787,10 +787,19 @@ class SubmitApp(Application):
         if config.database.provider == 'sqlite' and db in ('', ':memory:', None):
             raise ConfigurationError('Submitting tasks to in-memory database has no effect')
 
+    def check_tags(self: SubmitApp) -> None:
+        """Ensure valid tags."""
+        self.tags = {} if not self.taglist else Tag.parse_cmdline_list(self.taglist)
+        try:
+            Task.ensure_valid_tag(self.tags)
+        except (ValueError, TypeError) as error:
+            raise ArgumentError(str(error)) from error
+
     def __enter__(self: SubmitApp) -> SubmitApp:
         """Open file if not stdin."""
         self.source = sys.stdin if self.filepath == '-' else open(self.filepath, mode='r')
         self.check_config()
+        self.check_tags()
         if config.database.provider == 'sqlite' or self.auto_initdb:
             initdb()  # Auto-initialize if local sqlite provider
         else:

{hypershell-2.5.1 → hypershell-2.6.1}/src/hypershell/task.py

@@ -41,12 +41,13 @@ from hypershell.core.exceptions import handle_exception, handle_exception_silent
 from hypershell.core.logging import Logger, HOSTNAME
 from hypershell.core.remote import SSHConnection
 from hypershell.core.types import smart_coerce, JSONValue
+from hypershell.core.tag import Tag
 from hypershell.data.core import Session
 from hypershell.data.model import Task, to_json_type
 from hypershell.data import ensuredb
 
 # public interface
-__all__ = ['TaskGroupApp', 'Tag', ]
+__all__ = ['TaskGroupApp', ]
 
 # initialize logger
 log = Logger.with_name(__name__)
@@ -80,6 +81,7 @@ class TaskSubmitApp(Application):
     argv: List[str] = []
     interface.add_argument('argv', nargs='+')
 
+    tags: Dict[str, JSONValue] = {}
     taglist: List[str] = []
     interface.add_argument('-t', '--tag', nargs='*', dest='taglist')
 
@@ -90,11 +92,19 @@ class TaskSubmitApp(Application):
     def run(self: TaskSubmitApp) -> None:
         """Submit task to database."""
         ensuredb()
-        task = Task.new(args=' '.join(self.argv),
-                        tag=(None if not self.taglist else Tag.parse_cmdline_list(self.taglist)))
+        self.check_tags()
+        task = Task.new(args=' '.join(self.argv), tag=self.tags)
         Task.add(task)
         print(task.id)
 
+    def check_tags(self: TaskSubmitApp) -> None:
+        """Ensure valid tags."""
+        self.tags = {} if not self.taglist else Tag.parse_cmdline_list(self.taglist)
+        try:
+            Task.ensure_valid_tag(self.tags)
+        except (ValueError, TypeError) as error:
+            raise ArgumentError(str(error)) from error
+
 
 # Catch bad UUID before we touch the database
 UUID_PATTERN: re.Pattern = re.compile(
@@ -860,7 +870,8 @@ class TaskUpdateApp(Application, SearchableMixin):
                 print(f'Stopping (invalid response: "{response}")')
                 return
 
-        if self.delete_mode:
+        # Delete is handled later if a --limit is used with search
+        if self.delete_mode and self.limit is None:
             query.delete()
             Session.commit()
             log.info(f'Deleted {count} tasks')
@@ -895,25 +906,27 @@ class TaskUpdateApp(Application, SearchableMixin):
             # We cannot apply an UPDATE query with a LIMIT field
             # The alternative is to pull the data and batch the update
             # While terribly inefficient at least it has a LIMIT
+            tasks = query.all()
+            tasks_it = iter(tasks)
+            if self.delete_mode:
+                while batch := tuple(itertools.islice(tasks_it, 100)):
+                    Task.delete_all(list(batch))
             if field_updates:
-                tasks = query.all()
-                tasks_it = iter(tasks)
                 while batch := tuple(itertools.islice(tasks_it, 100)):
                     Task.update_all([{'id': task.id, **field_updates} for task in batch])
             if tag_updates:
-                tasks = query.all()
-                tasks_it = iter(tasks)
                 while batch := tuple(itertools.islice(tasks_it, 100)):
                     Task.update_all([{'id': task.id, 'tag': {**task.tag, **tag_updates}} for task in batch])
             if self.remove_tag:
-                tasks = query.all()
-                tasks_it = iter(tasks)
                 while batch := tuple(itertools.islice(tasks_it, 100)):
                     Task.update_all([
                         {'id': task.id, 'tag': self.drop_items(task.tag, *self.remove_tag)}
                         for task in batch
                     ])
-            log.info(f'Updated {count} tasks')
+            if self.delete_mode:
+                log.info(f'Deleted {count} tasks')
+            else:
+                log.info(f'Updated {count} tasks')
             return
 
         if field_updates:
@@ -1034,11 +1047,9 @@ class TaskGroupApp(ApplicationGroup):
     """Search, submit, track, and manage individual tasks."""
 
     interface = Interface(TASK_PROGRAM, TASK_USAGE, TASK_HELP)
-
-    interface.add_argument('command')
     interface.add_argument('--list-columns', action='version', version=' '.join(Task.columns))
+    interface.add_argument('command')
 
-    command = None
     commands = {
         'submit': TaskSubmitApp,
         'info': TaskInfoApp,
@@ -1096,34 +1107,6 @@ class WhereClause:
             raise ArgumentError(f'Where clause not understood ({argument})')
 
 
-@dataclass
-class Tag:
-    """Tag specification."""
-
-    name: str
-    value: JSONValue = ''
-
-    def to_dict(self: Tag) -> Dict[str, JSONValue]:
-        """Format tag specification as dictionary."""
-        return {self.name: self.value, }
-
-    @classmethod
-    def from_cmdline(cls: Type[Tag], arg: str) -> Tag:
-        """Construct from command-line `arg`."""
-        tag_part = arg.strip().split(':', 1)
-        if len(tag_part) == 1:
-            return cls(name=tag_part[0].strip())
-        else:
-            name, value = tag_part[0].strip(), smart_coerce(tag_part[1].strip())
-            Task.ensure_valid_tag({name: value})
-            return cls(name, value)
-
-    @classmethod
-    def parse_cmdline_list(cls: Type[Tag], args: List[str]) -> Dict[str, Optional[JSONValue]]:
-        """Parse command-line list of tags."""
-        return {tag.name: tag.value for tag in map(cls.from_cmdline, args)}
-
-
 def print_normal(task: Task) -> None:
     """Print semi-structured task metadata with all field names."""
     task_data = {k: json.dumps(to_json_type(v)).strip('"') for k, v in task.to_dict().items()}