solid-jobs-py 0.1.0__py3-none-any.whl

solid_jobs/__init__.py

@@ -0,0 +1,28 @@
+"""SolidJobs for Python: Redis task runner on PEP 734 subinterpreters.
+
+Redis access goes through the Ruby ``solid-redis`` gem, run as a sidecar.
+"""
+
+from . import lab
+from .blueprint import Blueprint
+from .catalog import (Channel, Claims, Counters, DiscardedTasks, Execution, Executions, Node, Nodes,
+                      PlannedTasks, RetryingTasks, StoredTask)
+from .conductor import Conductor
+from .errors import (ConfigurationError, Error, ExecutionHalt, GatewayError, InvalidArgumentError,
+                     InvalidJobError, NoCapturedTask, RedisCommandError, RedisConnectionError, RedisError)
+from .integrity_check import IntegrityCheck
+from .publisher import Publisher
+from .settings import config, configure, reset, use_config
+from .task import Task
+from .version import VERSION
+
+__version__ = VERSION
+testing = lab.testing
+
+__all__ = [
+    "Blueprint", "Channel", "Claims", "Conductor", "ConfigurationError", "Counters", "DiscardedTasks",
+    "Error", "Execution", "ExecutionHalt", "Executions", "GatewayError", "IntegrityCheck",
+    "InvalidArgumentError", "InvalidJobError", "NoCapturedTask", "Node", "Nodes", "PlannedTasks",
+    "Publisher", "RedisCommandError", "RedisConnectionError", "RedisError", "RetryingTasks",
+    "StoredTask", "Task", "VERSION", "config", "configure", "lab", "reset", "testing", "use_config",
+]

solid_jobs/__main__.py

@@ -0,0 +1,5 @@
+import sys
+
+from .console import main
+
+sys.exit(main())

solid_jobs/blueprint.py

@@ -0,0 +1,164 @@
+import importlib
+import json
+import logging
+import os
+import secrets
+import socket
+
+from . import gateway
+from .errors import ConfigurationError
+from .instrumentation import NullInstrumenter
+from .interceptors import InterceptorRegistry
+from .ipc import GatewayPool
+from .utilities import class_path, constantize, stringify_keys
+
+DEFAULT_TASK_OPTIONS = {"channel": "default", "max_failures": 25}
+CHANNEL_ORDERS = ("priority", "weighted", "shuffle")
+
+
+def _normalize_channels(values):
+    if isinstance(values, str):
+        values = [values]
+    entries = []
+    for value in values:
+        name, weight = value if isinstance(value, (list, tuple)) else (value, 1)
+        name, weight = str(name), int(weight)
+        if weight <= 0:
+            raise ValueError("Channel weight must be positive")
+        entries.append((name, weight))
+    return entries
+
+
+class Blueprint:
+    """Runtime configuration. Redis is reached through the Ruby solid-redis sidecar."""
+
+    def __init__(self, redis_url=None, concurrency=5, channels=("default",), gateway_socket=None):
+        self.redis_url = redis_url or os.environ.get("REDIS_URL", "redis://127.0.0.1:6379/0")
+        self.gateway_socket = gateway_socket or os.environ.get("SOLID_JOBS_GATEWAY_SOCKET")
+        self.ruby_command = tuple(os.environ.get("SOLID_JOBS_RUBY", "ruby").split())
+        self.concurrency = int(concurrency)
+        self._channels = _normalize_channels(channels)
+        self._default_task_options = dict(DEFAULT_TASK_OPTIONS)
+        self.publish_interceptors = InterceptorRegistry()
+        self.execute_interceptors = InterceptorRegistry()
+        self.on_complex_arguments = "raise"
+        self.poll_interval_average = 5.0
+        self.shutdown_timeout = 25.0
+        self._channel_order = "weighted"
+        self.reliable_fetch = True
+        self.retry_base_delay = 15.0
+        self.retry_max_delay = 3_600.0
+        self.discarded_limit = 10_000
+        self.discarded_retention = 180 * 24 * 60 * 60
+        self.logger = logging.getLogger("solid_jobs")
+        self.identity = f"{socket.gethostname()}:{os.getpid()}:{secrets.token_hex(6)}"
+        self.instrumenter = NullInstrumenter()
+        self.requires = []
+        self._callbacks = {}
+        self._redis_pool = None
+
+    @property
+    def redis_pool(self):
+        if self._redis_pool is None:
+            path = self.gateway_socket or gateway.ensure(self.redis_url, self.ruby_command)
+            self._redis_pool = GatewayPool(path)
+        return self._redis_pool
+
+    def close(self):
+        if self._redis_pool is not None:
+            self._redis_pool.close()
+            self._redis_pool = None
+        return self
+
+    @property
+    def channels(self):
+        return [name for name, _ in self._channels]
+
+    @channels.setter
+    def channels(self, values):
+        self._channels = _normalize_channels(values)
+
+    @property
+    def channel_entries(self):
+        return list(self._channels)
+
+    @property
+    def channel_order(self):
+        return self._channel_order
+
+    @channel_order.setter
+    def channel_order(self, mode):
+        if mode not in CHANNEL_ORDERS:
+            raise ValueError("channel_order must be 'priority', 'weighted', or 'shuffle'")
+        self._channel_order = mode
+
+    @property
+    def default_task_options(self):
+        return dict(self._default_task_options)
+
+    @default_task_options.setter
+    def default_task_options(self, options):
+        self._default_task_options = {**DEFAULT_TASK_OPTIONS, **stringify_keys(options)}
+
+    def on(self, event, callback):
+        self._callbacks.setdefault(event, []).append(callback)
+        return self
+
+    def fire(self, event, *arguments):
+        for callback in self._callbacks.get(event, ()):
+            callback(*arguments)
+
+    def __repr__(self):
+        return f"<Blueprint concurrency={self.concurrency} channels={self.channels!r}>"
+
+    def snapshot(self):
+        """JSON string a subinterpreter uses to rebuild an equivalent configuration."""
+        if self.gateway_socket is None:
+            self.gateway_socket = gateway.ensure(self.redis_url, self.ruby_command)
+        return json.dumps({
+            "redis_url": self.redis_url,
+            "gateway_socket": self.gateway_socket,
+            "concurrency": self.concurrency,
+            "channels": self._channels,
+            "channel_order": self._channel_order,
+            "reliable_fetch": self.reliable_fetch,
+            "default_task_options": self._default_task_options,
+            "on_complex_arguments": self.on_complex_arguments,
+            "poll_interval_average": self.poll_interval_average,
+            "shutdown_timeout": self.shutdown_timeout,
+            "discarded_limit": self.discarded_limit,
+            "discarded_retention": self.discarded_retention,
+            "retry_base_delay": self.retry_base_delay,
+            "retry_max_delay": self.retry_max_delay,
+            "publish_interceptors": self.publish_interceptors.export(),
+            "execute_interceptors": self.execute_interceptors.export(),
+            "identity": self.identity,
+            "instrumenter": class_path(type(self.instrumenter)),
+            "requires": list(self.requires),
+            "log_level": self.logger.getEffectiveLevel(),
+        })
+
+    @classmethod
+    def from_snapshot(cls, raw):
+        data = json.loads(raw)
+        config = cls(
+            redis_url=data["redis_url"], concurrency=data["concurrency"],
+            channels=data["channels"], gateway_socket=data["gateway_socket"],
+        )
+        config.channel_order = data["channel_order"]
+        for name in ("reliable_fetch", "on_complex_arguments", "poll_interval_average", "shutdown_timeout",
+                     "discarded_limit", "discarded_retention", "retry_base_delay", "retry_max_delay"):
+            setattr(config, name, data[name])
+        config.default_task_options = data["default_task_options"]
+        config.identity = data["identity"]
+        config.requires = data["requires"]
+        config.logger.setLevel(data["log_level"])
+        for module in config.requires:
+            importlib.import_module(module)
+        config.publish_interceptors.import_(data["publish_interceptors"])
+        config.execute_interceptors.import_(data["execute_interceptors"])
+        try:
+            config.instrumenter = constantize(data["instrumenter"])()
+        except Exception as error:
+            raise ConfigurationError(f"instrumenter must be a no-argument class: {error}") from error
+        return config

solid_jobs/catalog.py

@@ -0,0 +1,285 @@
+"""Read/inspect API over the SolidJobs Redis keyspace (counters, channels, nodes, claims...)."""
+
+import json
+from datetime import datetime
+
+from . import keyspace, settings
+from .publisher import Publisher
+
+
+def _cfg(config):
+    return config or settings.config()
+
+
+def _scan(pool, pattern):
+    cursor, found = "0", []
+    while True:
+        cursor, batch = pool.call("SCAN", cursor, "MATCH", pattern, "COUNT", 100)
+        found.extend(batch)
+        if cursor == "0":
+            return found
+
+
+def _time(seconds):
+    return datetime.fromtimestamp(float(seconds)) if seconds is not None else None
+
+
+def _pairs(flat):
+    flat = flat or []
+    return list(zip(flat[0::2], flat[1::2]))
+
+
+class Counters:
+    def __init__(self, config=None):
+        self.config = _cfg(config)
+
+    @property
+    def _pool(self):
+        return self.config.redis_pool
+
+    def completed(self):
+        return int(self._pool.call("GET", keyspace.PROCESSED) or 0)
+
+    def failed(self):
+        return int(self._pool.call("GET", keyspace.FAILED) or 0)
+
+    def ready(self):
+        return sum(channel.size() for channel in Channel.catalog(self.config))
+
+    def planned(self):
+        return int(self._pool.call("ZCARD", keyspace.PLANNED))
+
+    def retrying(self):
+        return int(self._pool.call("ZCARD", keyspace.RETRIES))
+
+    def discarded(self):
+        return int(self._pool.call("ZCARD", keyspace.DISCARDED))
+
+    def nodes(self):
+        return int(self._pool.call("SCARD", keyspace.NODES))
+
+    def executions(self):
+        return sum(node.active_count() for node in Nodes(self.config))
+
+    def reset(self):
+        self._pool.call("DEL", keyspace.PROCESSED, keyspace.FAILED)
+        return self
+
+
+class StoredTask:
+    def __init__(self, raw, score=None, config=None, owner=None):
+        self.raw = raw
+        self.envelope = json.loads(raw)
+        self.score = float(score) if score is not None else None
+        self.config = _cfg(config)
+        self.owner = owner
+
+    id = property(lambda self: self.envelope.get("id"))
+    channel = property(lambda self: self.envelope.get("channel"))
+    task = property(lambda self: self.envelope.get("task"))
+    arguments = property(lambda self: self.envelope.get("arguments"))
+    created_at = property(lambda self: self._millis("created_ms"))
+    queued_at = property(lambda self: self._millis("queued_ms"))
+    due_at = property(lambda self: _time(self.score))
+
+    def remove(self):
+        if isinstance(self.owner, TimedTasks):
+            return self.config.redis_pool.call("ZREM", self.owner.key, self.raw)
+        if isinstance(self.owner, Channel):
+            return self.config.redis_pool.call("LREM", self.owner.storage_key, 1, self.raw)
+        return False
+
+    def requeue(self):
+        self.remove()
+        return Publisher(config=self.config).publish(self.envelope)
+
+    def _millis(self, field):
+        value = self.envelope.get(field)
+        return _time(float(value) / 1000) if value else None
+
+
+class Channel:
+    def __init__(self, name, config=None):
+        self.name = str(name)
+        self.config = _cfg(config)
+
+    @classmethod
+    def catalog(cls, config=None):
+        config = _cfg(config)
+        return [cls(name, config) for name in sorted(config.redis_pool.call("SMEMBERS", keyspace.CHANNELS) or [])]
+
+    @property
+    def storage_key(self):
+        return keyspace.channel(self.name)
+
+    def __iter__(self):
+        for raw in reversed(self.config.redis_pool.call("LRANGE", self.storage_key, 0, -1) or []):
+            yield StoredTask(raw, config=self.config, owner=self)
+
+    def size(self):
+        return int(self.config.redis_pool.call("LLEN", self.storage_key))
+
+    def waiting_time(self):
+        import time
+
+        raw = self.config.redis_pool.call("LINDEX", self.storage_key, -1)
+        if not raw:
+            return 0.0
+        envelope = json.loads(raw)
+        stamp = envelope.get("queued_ms") or envelope.get("created_ms")
+        return max(time.time() - float(stamp) / 1000, 0.0) if stamp else 0.0
+
+    def purge(self):
+        with self.config.redis_pool.pipelined() as pipeline:
+            pipeline.call("DEL", self.storage_key)
+            pipeline.call("SREM", keyspace.CHANNELS, self.name)
+        return self
+
+    def paused(self):
+        return int(self.config.redis_pool.call("SISMEMBER", keyspace.PAUSED_CHANNELS, self.name)) == 1
+
+    def pause(self):
+        self.config.redis_pool.call("SADD", keyspace.PAUSED_CHANNELS, self.name)
+        return self
+
+    def resume(self):
+        self.config.redis_pool.call("SREM", keyspace.PAUSED_CHANNELS, self.name)
+        return self
+
+
+class TimedTasks:
+    def __init__(self, key, config=None):
+        self.key = key
+        self.config = _cfg(config)
+
+    def __iter__(self):
+        values = self.config.redis_pool.call("ZRANGE", self.key, 0, -1, "WITHSCORES")
+        for raw, score in _pairs(values):
+            yield StoredTask(raw, score=score, config=self.config, owner=self)
+
+    def size(self):
+        return int(self.config.redis_pool.call("ZCARD", self.key))
+
+    def purge(self):
+        self.config.redis_pool.call("DEL", self.key)
+        return self
+
+    def locate(self, *ids):
+        wanted = {i for group in ids for i in (group if isinstance(group, (list, tuple, set)) else [group])}
+        return [task for task in self if task.id in wanted]
+
+    def requeue_all(self):
+        for task in list(self):
+            task.requeue()
+
+
+class PlannedTasks(TimedTasks):
+    def __init__(self, config=None):
+        super().__init__(keyspace.PLANNED, config)
+
+
+class RetryingTasks(TimedTasks):
+    def __init__(self, config=None):
+        super().__init__(keyspace.RETRIES, config)
+
+
+class DiscardedTasks(TimedTasks):
+    def __init__(self, config=None):
+        super().__init__(keyspace.DISCARDED, config)
+
+
+class Execution:
+    def __init__(self, node_id, executor_id, raw):
+        self.node_id = node_id
+        self.executor_id = executor_id
+        self.attributes = json.loads(raw)
+
+    channel = property(lambda self: self.attributes.get("channel"))
+    envelope = property(lambda self: self.attributes.get("envelope"))
+    claim_token = property(lambda self: self.attributes.get("claim_token"))
+    started_at = property(lambda self: _time(self.attributes["started_at"]))
+    attempt = property(lambda self: int(self.attributes.get("attempt") or 1))
+
+
+class Node:
+    CONTROL_SIGNALS = {"pause": "TSTP", "shutdown": "TERM", "backtraces": "TTIN"}
+
+    def __init__(self, identity, config=None):
+        self.identity = identity
+        self.config = _cfg(config)
+        self.attributes = dict(_pairs(self.config.redis_pool.call("HGETALL", keyspace.node(identity))))
+        self.info = json.loads(self.attributes.get("info", "{}"))
+
+    hostname = property(lambda self: self.info.get("hostname"))
+    pid = property(lambda self: int(self.info.get("pid") or 0))
+    channels = property(lambda self: self.info.get("channels") or [])
+
+    def active_count(self):
+        return int(self.attributes.get("busy") or 0)
+
+    def concurrency(self):
+        return int(self.attributes.get("concurrency") or 0)
+
+    def quiet(self):
+        return self.attributes.get("quiet") == "true"
+
+    def last_seen_at(self):
+        return _time(self.attributes["beat"]) if "beat" in self.attributes else None
+
+    def request_control(self, action):
+        signal = self.CONTROL_SIGNALS[action]
+        self.config.redis_pool.call("LPUSH", keyspace.node_signals(self.identity), signal)
+        return True
+
+    def executions(self):
+        fields = self.config.redis_pool.call("HGETALL", keyspace.node_work(self.identity))
+        return [Execution(self.identity, executor, raw) for executor, raw in _pairs(fields)]
+
+
+class Nodes:
+    def __init__(self, config=None):
+        self.config = _cfg(config)
+
+    def __iter__(self):
+        for identity in self.config.redis_pool.call("SMEMBERS", keyspace.NODES) or []:
+            node = Node(identity, self.config)
+            if node.last_seen_at() is not None:
+                yield node
+            else:
+                self.config.redis_pool.call("SREM", keyspace.NODES, identity)
+
+
+class ClaimInfo:
+    def __init__(self, raw):
+        self.attributes = json.loads(raw)
+
+    task_id = property(lambda self: self.attributes.get("task_id"))
+    claim_token = property(lambda self: self.attributes.get("claim_token"))
+    node_id = property(lambda self: self.attributes.get("node_id"))
+    executor_id = property(lambda self: self.attributes.get("executor_id"))
+    channel = property(lambda self: self.attributes.get("channel"))
+    attempt = property(lambda self: int(self.attributes["attempt"]))
+    claimed_at = property(lambda self: _time(self.attributes["claimed_at"]))
+
+
+class Claims:
+    def __init__(self, config=None):
+        self.config = _cfg(config)
+
+    def __iter__(self):
+        pool = self.config.redis_pool
+        for key in _scan(pool, f"{keyspace.PREFIX}:node:*:claims"):
+            for raw in pool.call("HVALS", key) or []:
+                yield ClaimInfo(raw)
+
+    def find_task(self, task_id):
+        return next((claim for claim in self if claim.task_id == task_id), None)
+
+
+class Executions:
+    def __init__(self, config=None):
+        self.nodes = Nodes(config)
+
+    def __iter__(self):
+        for node in self.nodes:
+            yield from node.executions()