cronstable 1.2.11__py3-none-any.whl

cronstable/__main__.py

@@ -0,0 +1,278 @@
+import argparse
+import asyncio
+import logging
+import os
+import sys
+
+import cronstable.version
+from cronstable import platform
+
+# Where -c looks when not given: /etc/cronstable.d on POSIX,
+# %APPDATA%\cronstable on
+# Windows (see cronstable.platform).
+CONFIG_DEFAULT = platform.DEFAULT_CONFIG_PATH
+
+
+def _add_state_subcommands(parser: argparse.ArgumentParser) -> None:
+    """Wire the `cronstable state <action>` administration subcommands.
+
+    Bare `cronstable` (no subcommand) stays the daemon.  Each action accepts
+    its own -c/--config (same dest and default as the daemon flag) so both
+    `cronstable -c X state gc` and `cronstable state gc -c X` work.
+    """
+    sub = parser.add_subparsers(dest="command", metavar="COMMAND")
+    state = sub.add_parser(
+        "state",
+        help="administer the durable state store (backup/restore/migrate/"
+        "gc/check/migrate-schema)",
+    )
+    actions = state.add_subparsers(dest="state_command", metavar="ACTION")
+
+    def _with_config(sub_parser):
+        # SUPPRESS, not CONFIG_DEFAULT: a subparser default would otherwise
+        # OVERWRITE a root-level `cronstable -c X state ...` value (argparse
+        # applies subparser defaults after the root parse). The root parser
+        # already supplies the default.
+        sub_parser.add_argument(
+            "-c",
+            "--config",
+            default=argparse.SUPPRESS,
+            metavar="FILE-OR-DIR",
+            help="configuration with the `state:` section to administer",
+        )
+        return sub_parser
+
+    backup = _with_config(
+        actions.add_parser(
+            "backup", help="write a .tar.gz backup of the store"
+        )
+    )
+    backup.add_argument("-o", "--output", required=True, metavar="FILE.tar.gz")
+    restore = _with_config(
+        actions.add_parser("restore", help="restore a backup into the store")
+    )
+    restore.add_argument("archive", metavar="FILE.tar.gz")
+    restore.add_argument(
+        "--force",
+        default=False,
+        action="store_true",
+        help="merge into a non-empty store (NOT safe while a daemon uses it)",
+    )
+    migrate = _with_config(
+        actions.add_parser(
+            "migrate",
+            help="copy the store to another path or mount "
+            "(local disk <-> S3 Files / EFS)",
+        )
+    )
+    migrate.add_argument("--dest", required=True, metavar="PATH")
+    migrate.add_argument(
+        "--dest-deployment-id",
+        default=None,
+        metavar="ID",
+        help="namespace at the destination (default: keep the current one)",
+    )
+    migrate.add_argument(
+        "--force",
+        default=False,
+        action="store_true",
+        help="overwrite a non-empty destination store",
+    )
+    gc = _with_config(
+        actions.add_parser(
+            "gc", help="garbage-collect state of unreferenced jobs"
+        )
+    )
+    gc.add_argument("--dry-run", default=False, action="store_true")
+    _with_config(
+        actions.add_parser(
+            "check", help="verify the store is usable and print an inventory"
+        )
+    )
+    migrate_schema = _with_config(
+        actions.add_parser(
+            "migrate-schema",
+            help="rewrite records of older known record schemes",
+        )
+    )
+    migrate_schema.add_argument(
+        "--dry-run", default=False, action="store_true"
+    )
+
+    # The job-facing state commands. The KV actions (get/set/delete/
+    # keys) hang off the SAME `state` subparser as the admin actions above and
+    # coexist with them (the action name routes); the other verbs (cursor/
+    # lock/artifact/idempotent/secret) are their own top-level commands. Both
+    # are thin clients of the daemon's loopback endpoint. Imported here (not at
+    # module load) so the import cost is paid only when the CLI is built.
+    from cronstable import jobcli
+
+    jobcli.add_state_job_actions(actions)
+    jobcli.add_job_commands(sub)
+
+
+def main_loop(loop):
+    parser = argparse.ArgumentParser(prog="cronstable")
+    parser.add_argument(
+        "-c",
+        "--config",
+        default=CONFIG_DEFAULT,
+        metavar="FILE-OR-DIR",
+        help="configuration file, or directory containing configuration files",
+    )
+    parser.add_argument("-l", "--log-level", default="INFO")
+    parser.add_argument(
+        "-v", "--validate-config", default=False, action="store_true"
+    )
+    parser.add_argument(
+        "--job-set-id",
+        default=False,
+        action="store_true",
+        help="print the job-set id (an order-independent hash of every job's "
+        "effective configuration) and exit; identical across instances "
+        "running the same set of jobs",
+    )
+    parser.add_argument("--version", default=False, action="store_true")
+    _add_state_subcommands(parser)
+    # `lock run NAME [flags] -- CMD...` carries an arbitrary trailing command.
+    # argparse cannot capture it portably: nargs=REMAINDER swallows our own
+    # flags into the command list, while nargs="*" only picks up the tokens
+    # after "--" on Python >= 3.13 (older argparse reports them as
+    # "unrecognized arguments"). So split the command off at the first "--"
+    # ourselves -- identical on every supported Python -- and hand argparse
+    # only the head, where our flags and NAME parse cleanly everywhere.
+    argv = sys.argv[1:]
+    trailing_command = None
+    if "--" in argv:
+        cut = argv.index("--")
+        argv, trailing_command = argv[:cut], argv[cut + 1 :]
+    args = parser.parse_args(argv)
+    if trailing_command is not None:
+        if (
+            getattr(args, "command", None) == "lock"
+            and getattr(args, "lock_command", None) == "run"
+        ):
+            args.run_command = trailing_command
+        else:
+            parser.error("`--` is only valid before a `lock run` command")
+
+    logging.basicConfig(level=getattr(logging, args.log_level))
+    # logging.getLogger("asyncio").setLevel(logging.WARNING)
+    logger = logging.getLogger("cronstable")
+
+    if args.version:
+        print(cronstable.version.version)
+        sys.exit(0)
+
+    command = getattr(args, "command", None)
+    if command == "state":
+        # `state get/set/delete/keys` are job-facing (they reach the running
+        # daemon's loopback endpoint); everything else under `state` is offline
+        # store administration. Route by action name so the two coexist.
+        from cronstable import jobcli
+
+        if getattr(args, "state_command", None) in jobcli.STATE_JOB_ACTIONS:
+            sys.exit(jobcli.dispatch(args))
+        # lazy import: the admin module (tarfile etc.) costs the daemon and
+        # the stateless install nothing.
+        from cronstable import state_admin
+
+        sys.exit(state_admin.dispatch(args))
+
+    if command in (
+        "cursor",
+        "lock",
+        "artifact",
+        "idempotent",
+        "secret",
+        "xcom",
+    ):
+        from cronstable import jobcli
+
+        sys.exit(jobcli.dispatch(args))
+
+    if args.config == CONFIG_DEFAULT and not os.path.exists(args.config):
+        print(
+            "cronstable error: configuration file not found, please provide "
+            "one with the --config option",
+            file=sys.stderr,
+        )
+        parser.print_help(sys.stderr)
+        sys.exit(1)
+
+    # Imported here, not at module top: this pulls in aiohttp, strictyaml,
+    # sentry_sdk and the rest of the daemon graph (~300ms of import). The
+    # branches that exit before this point -- --version and the state / xcom
+    # / lock / cursor / artifact / idempotent / secret subcommands (thin
+    # urllib clients of the running daemon, routinely spawned from inside
+    # jobs) -- never touch Cron, so a job-facing CLI call no longer pays that
+    # cost. Everything from here down (the daemon, --job-set-id,
+    # --validate-config) needs it.
+    from cronstable.cron import ConfigError, Cron
+
+    try:
+        cron = Cron(args.config)
+    except ConfigError as err:
+        logger.error("Configuration error: %s", str(err))
+        sys.exit(1)
+
+    if args.job_set_id:
+        print(cron.job_set_id())
+        sys.exit(0)
+
+    if args.validate_config:
+        logger.info("Configuration is valid.")
+        sys.exit(0)
+
+    # Wire Ctrl-C / termination to a graceful shutdown.  The mechanism differs
+    # per platform (loop signal handlers on POSIX, signal.signal on Windows),
+    # so it lives behind platform.install_shutdown_handlers.
+    remove_shutdown_handlers = platform.install_shutdown_handlers(
+        loop, cron.signal_shutdown
+    )
+    try:
+        loop.run_until_complete(cron.run())
+    finally:
+        remove_shutdown_handlers()
+
+
+def _new_event_loop():  # pragma: no cover
+    """The event loop to run on: uvloop's faster libuv loop when available,
+    otherwise stock asyncio.
+
+    uvloop is a drop-in, libuv-based replacement for asyncio's selector loop
+    that runs cronstable's I/O paths -- cluster gossip/lease HTTP, the web
+    dashboard, the Prometheus scrape -- markedly faster. It is strictly
+    optional (install the ``speedups`` extra to pull it in): it has no Windows
+    build (where cronstable also needs the Proactor loop for subprocess
+    support)
+    and ships no wheels for some of the leaner architectures we target, so a
+    missing or unimportable uvloop silently falls back to stock asyncio with
+    identical behavior. Selecting the loop directly (rather than via
+    ``asyncio.set_event_loop_policy``) sidesteps the event-loop-policy API that
+    Python 3.14 deprecates.
+
+    ``asyncio.new_event_loop()`` yields the right stock loop per platform: a
+    subprocess-capable Proactor loop on Windows (the default since 3.8) and a
+    selector loop on POSIX.
+    """
+    if sys.platform != "win32":
+        try:
+            import uvloop
+        except ImportError:
+            pass
+        else:
+            return uvloop.new_event_loop()
+    return asyncio.new_event_loop()
+
+
+def main():  # pragma: no cover
+    _loop = _new_event_loop()
+    try:
+        main_loop(_loop)
+    finally:
+        _loop.close()
+
+
+if __name__ == "__main__":  # pragma: no cover
+    main()

cronstable/_json.py

@@ -0,0 +1,127 @@
+"""Optional orjson acceleration for the hot JSON serialization paths.
+
+orjson (a compiled Rust/pyo3 extension) serializes and parses JSON several
+times faster than the stdlib and hands back ``bytes`` directly, saving the
+separate ``.encode("utf-8")`` every persistence site does by hand. It is an
+OPTIONAL speedup, wired exactly like the uvloop event-loop swap: it ships no
+wheels for some of the leaner architectures cronstable targets (riscv64, armv6,
+musl ppc64le/s390x), so it stays out of the core dependency set (install the
+``speedups`` extra to pull it in) and this module transparently falls back to
+the stdlib ``json`` when it is absent -- identical behaviour, just slower.
+
+CONTRACT -- read before pointing anything new at these helpers:
+
+Use them only where the serialized bytes are round-tripped back through
+:func:`loads` (durable state records, leases, documents; parsing peer gossip
+bodies). orjson always emits UTF-8 and does NOT ``ensure_ascii``, so for
+non-ASCII input its bytes are NOT identical to
+``json.dumps(...).encode("utf-8")``. Anywhere the exact bytes matter -- a value
+fed into a hash / content-address, or compared across nodes or versions (the
+job-set fingerprint in :mod:`cronstable.fingerprint`, the cluster peer ETag in
+:mod:`cronstable.cluster`) -- keep the stdlib ``json`` directly, so the output
+stays stable and backend-independent whether or not a given host has orjson.
+"""
+
+import json as _stdlib
+import math
+from typing import Any, Union, cast
+
+try:
+    import orjson
+except ImportError:  # pragma: no cover - exercised on the no-orjson baseline
+    orjson = None  # type: ignore[assignment]
+
+
+# The portable-value contract shared by BOTH backends.  orjson and the stdlib
+# are not interchangeable at the edges: orjson accepts only integers in the
+# signed-64..unsigned-64 window (it raises on anything wider) and SILENTLY
+# rewrites a non-finite float to ``null``; the stdlib accepts arbitrary-width
+# ints and writes ``NaN`` / ``Infinity`` tokens that orjson then refuses to
+# parse.  So a value outside this window is serialized DIFFERENTLY -- or
+# unreadably -- depending on whether a given host installed the optional orjson
+# extra.  On a mixed fleet sharing one store that is silent corruption: a node
+# without orjson writes ``{"value":Infinity}`` that every orjson node's
+# ``loads`` then rejects (a wedged cursor / a watermark read back as unset ->
+# reprocessed backlog), or an orjson node coerces ``nan`` to ``null`` a stdlib
+# node reads back as ``None``.  :func:`dumps_bytes` rejects such values
+# UNIFORMLY, on every backend, so the durable store stays backend-independent:
+# bytes any node writes, every node can read.
+_INT_MIN = -(2**63)
+_INT_MAX = 2**64 - 1
+
+
+class UnsupportedValue(ValueError):
+    """A value that cannot be encoded identically across a mixed-orjson fleet.
+
+    Raised by :func:`dumps_bytes` (and available as a standalone pre-check via
+    :func:`ensure_portable`) for a non-finite float (``NaN`` / ``Infinity``),
+    an integer outside the 64-bit window orjson supports, or a non-string
+    object key.  Rejecting at write time is what keeps a record readable by
+    every node regardless of which ones have orjson.
+    """
+
+
+def ensure_portable(obj: Any) -> None:
+    """Raise :class:`UnsupportedValue` if ``obj`` is not fleet-portable JSON.
+
+    A recursive, backend-independent pre-check so the accept/reject decision is
+    identical on every host -- with or without orjson -- instead of one backend
+    silently corrupting what the other rejects.  Called at the top of
+    :func:`dumps_bytes`; a boundary that wants the check without serializing
+    (to translate the failure into its own error type) invokes it directly.
+    """
+    if isinstance(obj, bool):
+        return  # a bool is an int subclass but always in range and portable
+    if isinstance(obj, float):
+        if not math.isfinite(obj):
+            raise UnsupportedValue(
+                "non-finite float {!r} is not portable across the fleet "
+                "(NaN/Infinity serializes differently with and without "
+                "orjson)".format(obj)
+            )
+    elif isinstance(obj, int):
+        if obj < _INT_MIN or obj > _INT_MAX:
+            raise UnsupportedValue(
+                "integer {} is outside the portable signed/unsigned 64-bit "
+                "range [{}, {}]".format(obj, _INT_MIN, _INT_MAX)
+            )
+    elif isinstance(obj, dict):
+        for key, value in obj.items():
+            if not isinstance(key, str):
+                raise UnsupportedValue(
+                    "non-string object key {!r} is not portable across the "
+                    "fleet (orjson rejects it, the stdlib coerces it)".format(
+                        key
+                    )
+                )
+            ensure_portable(value)
+    elif isinstance(obj, (list, tuple)):
+        for value in obj:
+            ensure_portable(value)
+
+
+if orjson is not None:
+
+    def dumps_bytes(obj: Any, *, sort_keys: bool = False) -> bytes:
+        """Serialize ``obj`` to compact UTF-8 JSON bytes."""
+        ensure_portable(obj)
+        option = orjson.OPT_SORT_KEYS if sort_keys else 0
+        # cast: the tox mypy env runs --ignore-missing-imports (orjson not a
+        # dev dep), so orjson.dumps reads as Any and warn_return_any fires.
+        return cast(bytes, orjson.dumps(obj, option=option))
+
+    def loads(data: Union[bytes, str]) -> Any:
+        """Parse JSON from ``bytes`` or ``str``."""
+        return orjson.loads(data)
+
+else:
+
+    def dumps_bytes(obj: Any, *, sort_keys: bool = False) -> bytes:
+        """Serialize ``obj`` to compact UTF-8 JSON bytes."""
+        ensure_portable(obj)
+        text = _stdlib.dumps(obj, separators=(",", ":"), sort_keys=sort_keys)
+        return text.encode("utf-8")
+
+    def loads(data: Union[bytes, str]) -> Any:
+        """Parse JSON from ``bytes`` or ``str``."""
+        return _stdlib.loads(data)

cronstable/backends/__init__.py

@@ -0,0 +1,55 @@
+"""Optional lease-store leadership backends (kubernetes, etcd).
+
+Each module here implements :class:`cronstable.leadership.LeaseBackend` against
+a
+real coordination store, over plain HTTP via the core ``aiohttp`` dependency --
+so neither adds a runtime dependency and, by avoiding grpc/protobuf wheels,
+both run on every architecture cronstable targets.
+
+* **kubernetes** drives a ``coordination.k8s.io/v1`` ``Lease`` and has **two
+  interchangeable transports**: the official ``kubernetes`` client when it is
+  installed (and importable on this architecture), or a hand-rolled apiserver
+  REST transport otherwise.  ``cluster.kubernetes.clientLibrary`` chooses:
+  ``auto`` (default) prefers the native client and falls back to HTTP;
+  ``library`` requires it (a config error if absent); ``http`` forces the
+  hand-rolled path.
+* **etcd** uses etcd's own v3 gRPC-gateway JSON/HTTP API directly -- a single,
+  fully-portable transport, with no optional client library (the gateway is
+  etcd's first-class HTTP interface, so a native grpc client buys little).
+
+The modules are imported lazily by :func:`cronstable.leadership.make_backend`,
+so they never enter the import graph unless ``cluster.backend`` selects them.
+"""
+
+from cronstable.config import ConfigError
+
+# transport kinds returned by select_transport.
+TRANSPORT_HTTP = "http"
+TRANSPORT_LIBRARY = "library"
+
+
+def select_transport(
+    client_library: str, native_available: bool, backend: str
+) -> str:
+    """Choose the transport for a lease backend (pure; unit-tested).
+
+    ``client_library`` is the resolved ``cluster.<backend>.clientLibrary``
+    setting, ``native_available`` whether the native client imported on this
+    architecture.  ``auto`` prefers the native client when present; ``library``
+    requires it (raising :class:`~cronstable.config.ConfigError` if absent);
+    ``http`` always uses the hand-rolled transport.
+    """
+    if client_library == "http":
+        return TRANSPORT_HTTP
+    if client_library == "library":
+        if not native_available:
+            raise ConfigError(
+                "cluster.{0}.clientLibrary is 'library' but the native client "
+                "is not importable on this architecture; install the optional "
+                "cronstable[{0}] extra, or use clientLibrary auto/http".format(
+                    backend
+                )
+            )
+        return TRANSPORT_LIBRARY
+    # "auto": prefer the native client when present, else the HTTP fallback.
+    return TRANSPORT_LIBRARY if native_available else TRANSPORT_HTTP