taskito 0.4.0__tar.gz → 0.6.0__tar.gz

{taskito-0.4.0 → taskito-0.6.0}/Cargo.lock

@@ -69,9 +69,9 @@ checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
 
 [[package]]
 name = "cc"
-version = "1.2.56"
+version = "1.2.57"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2"
+checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423"
 dependencies = [
  "find-msvc-tools",
  "shlex",
@@ -194,9 +194,9 @@ dependencies = [
 
 [[package]]
 name = "diesel"
-version = "2.3.6"
+version = "2.3.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d9b6c2fc184a6fb6ebcf5f9a5e3bbfa84d8fd268cdfcce4ed508979a6259494d"
+checksum = "f4ae09a41a4b89f94ec1e053623da8340d996bc32c6517d325a9daad9b239358"
 dependencies = [
  "bitflags",
  "byteorder",
@@ -693,9 +693,9 @@ dependencies = [
 
 [[package]]
 name = "once_cell"
-version = "1.21.3"
+version = "1.21.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
+checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
 
 [[package]]
 name = "openssl-src"
@@ -708,9 +708,9 @@ dependencies = [
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.111"
+version = "0.9.112"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321"
+checksum = "57d55af3b3e226502be1526dfdba67ab0e9c96fc293004e79576b2b9edb0dbdb"
 dependencies = [
  "cc",
  "libc",
@@ -1199,9 +1199,20 @@ version = "0.12.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
 
+[[package]]
+name = "taskito-async"
+version = "0.6.0"
+dependencies = [
+ "async-trait",
+ "crossbeam-channel",
+ "pyo3",
+ "taskito-core",
+ "tokio",
+]
+
 [[package]]
 name = "taskito-core"
-version = "0.4.0"
+version = "0.6.0"
 dependencies = [
  "async-trait",
  "chrono",
@@ -1225,11 +1236,13 @@ dependencies = [
 
 [[package]]
 name = "taskito-python"
-version = "0.4.0"
+version = "0.6.0"
 dependencies = [
  "async-trait",
  "crossbeam-channel",
  "pyo3",
+ "serde_json",
+ "taskito-async",
  "taskito-core",
  "tokio",
  "uuid",

{taskito-0.4.0 → taskito-0.6.0}/PKG-INFO

@@ -1,11 +1,10 @@
 Metadata-Version: 2.4
 Name: taskito
-Version: 0.4.0
+Version: 0.6.0
 Classifier: Development Status :: 4 - Beta
 Classifier: Intended Audience :: Developers
 Classifier: License :: OSI Approved :: MIT License
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.9
 Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
@@ -14,6 +13,7 @@ Classifier: Programming Language :: Rust
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Classifier: Topic :: System :: Distributed Computing
 Requires-Dist: cloudpickle>=3.0
+Requires-Dist: boto3>=1.20 ; extra == 'aws'
 Requires-Dist: pytest>=7.0 ; extra == 'dev'
 Requires-Dist: pytest-asyncio>=0.21 ; extra == 'dev'
 Requires-Dist: pytest-cov>=4.0 ; extra == 'dev'
@@ -25,17 +25,20 @@ Requires-Dist: cryptography ; extra == 'encryption'
 Requires-Dist: fastapi>=0.100.0 ; extra == 'fastapi'
 Requires-Dist: pydantic>=2.0 ; extra == 'fastapi'
 Requires-Dist: flask>=2.0 ; extra == 'flask'
+Requires-Dist: google-cloud-storage>=2.0 ; extra == 'gcs'
 Requires-Dist: msgpack ; extra == 'msgpack'
 Requires-Dist: opentelemetry-api ; extra == 'otel'
 Requires-Dist: opentelemetry-sdk ; extra == 'otel'
 Requires-Dist: prometheus-client ; extra == 'prometheus'
 Requires-Dist: sentry-sdk ; extra == 'sentry'
+Provides-Extra: aws
 Provides-Extra: dev
 Provides-Extra: django
 Provides-Extra: docs
 Provides-Extra: encryption
 Provides-Extra: fastapi
 Provides-Extra: flask
+Provides-Extra: gcs
 Provides-Extra: msgpack
 Provides-Extra: otel
 Provides-Extra: postgres
@@ -44,7 +47,7 @@ Provides-Extra: redis
 Provides-Extra: sentry
 License-File: LICENSE
 Summary: Rust-powered task queue for Python. No broker required.
-Requires-Python: >=3.9
+Requires-Python: >=3.10
 Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM
 Project-URL: Changelog, https://github.com/pratyush618/taskito/blob/master/docs/changelog.md
 Project-URL: Documentation, https://taskito.grigori.in

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "taskito-core"
-version = "0.4.0"
+version = "0.6.0"
 edition = "2021"
 
 [features]

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/maintenance.rs

@@ -33,7 +33,7 @@ impl Scheduler {
 
         for job in stale_jobs {
             let error = format!("job timed out after {}ms", job.timeout_ms);
-            self.handle_result(JobResult::Failure {
+            let _ = self.handle_result(JobResult::Failure {
                 job_id: job.id.clone(),
                 error,
                 retry_count: job.retry_count,
@@ -41,6 +41,7 @@ impl Scheduler {
                 task_name: job.task_name.clone(),
                 wall_time_ns: 0,
                 should_retry: true,
+                timed_out: true,
             })?;
         }
 

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/mod.rs

@@ -60,6 +60,7 @@ pub enum JobResult {
         task_name: String,
         wall_time_ns: i64,
         should_retry: bool,
+        timed_out: bool,
     },
     Cancelled {
         job_id: String,
@@ -68,12 +69,45 @@ pub enum JobResult {
     },
 }
 
+/// Outcome of processing a job result, returned to the caller for
+/// Python-side middleware hook dispatch.
+#[derive(Debug, Clone)]
+pub enum ResultOutcome {
+    /// Task completed successfully.
+    Success { job_id: String, task_name: String },
+    /// Task failed and will be retried.
+    Retry {
+        job_id: String,
+        task_name: String,
+        error: String,
+        retry_count: i32,
+        timed_out: bool,
+    },
+    /// Task exhausted retries and moved to the dead-letter queue.
+    DeadLettered {
+        job_id: String,
+        task_name: String,
+        error: String,
+        timed_out: bool,
+    },
+    /// Task was cancelled during execution.
+    Cancelled { job_id: String, task_name: String },
+}
+
 /// Per-task configuration for retry, rate limiting, and circuit breaker.
 #[derive(Debug, Clone)]
 pub struct TaskConfig {
     pub retry_policy: RetryPolicy,
     pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
     pub circuit_breaker: Option<CircuitBreakerConfig>,
+    pub max_concurrent: Option<i32>,
+}
+
+/// Per-queue configuration for rate limiting and concurrency caps.
+#[derive(Debug, Clone)]
+pub struct QueueConfig {
+    pub rate_limit: Option<crate::resilience::rate_limiter::RateLimitConfig>,
+    pub max_concurrent: Option<i32>,
 }
 
 /// The central scheduler that coordinates job dispatch, retries, rate limiting, and circuit breakers.
@@ -83,6 +117,7 @@ pub struct Scheduler {
     dlq: DeadLetterQueue,
     circuit_breaker: CircuitBreaker,
     task_configs: HashMap<String, TaskConfig>,
+    queue_configs: HashMap<String, QueueConfig>,
     queues: Vec<String>,
     config: SchedulerConfig,
     shutdown: Arc<Notify>,
@@ -109,6 +144,7 @@ impl Scheduler {
             dlq,
             circuit_breaker,
             task_configs: HashMap::new(),
+            queue_configs: HashMap::new(),
             queues,
             config,
             shutdown: Arc::new(Notify::new()),
@@ -124,6 +160,10 @@ impl Scheduler {
         self.shutdown.clone()
     }
 
+    pub fn register_queue_config(&mut self, queue_name: String, config: QueueConfig) {
+        self.queue_configs.insert(queue_name, config);
+    }
+
     pub fn register_task(&mut self, task_name: String, config: TaskConfig) {
         if let Some(ref cb_config) = config.circuit_breaker {
             if let Err(e) = self.circuit_breaker.register(&task_name, cb_config) {
@@ -276,6 +316,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -290,6 +331,7 @@ mod tests {
                 task_name: "retry_task".to_string(),
                 wall_time_ns: 500_000,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -312,6 +354,7 @@ mod tests {
                 task_name: "exhausted_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: true,
+                timed_out: false,
             })
             .unwrap();
 
@@ -337,6 +380,7 @@ mod tests {
                 task_name: "no_retry_task".to_string(),
                 wall_time_ns: 100,
                 should_retry: false,
+                timed_out: false,
             })
             .unwrap();
 
@@ -373,6 +417,7 @@ mod tests {
                     refill_rate: 0.0,
                 }),
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 
@@ -414,6 +459,7 @@ mod tests {
                 retry_policy: RetryPolicy::default(),
                 rate_limit: None,
                 circuit_breaker: Some(cb_config),
+                max_concurrent: None,
             },
         );
 
@@ -452,6 +498,7 @@ mod tests {
                 },
                 rate_limit: None,
                 circuit_breaker: None,
+                max_concurrent: None,
             },
         );
 

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/poller.rs

@@ -14,6 +14,9 @@ const CIRCUIT_BREAKER_RETRY_DELAY_MS: i64 = 5000;
 /// Delay before re-scheduling a rate-limited job (ms).
 const RATE_LIMIT_RETRY_DELAY_MS: i64 = 1000;
 
+/// Delay before re-scheduling a concurrency-limited job (ms).
+const CONCURRENCY_RETRY_DELAY_MS: i64 = 500;
+
 impl Scheduler {
     pub(super) fn try_dispatch(&self, job_tx: &tokio::sync::mpsc::Sender<Job>) -> Result<bool> {
         let now = now_millis();
@@ -53,6 +56,26 @@ impl Scheduler {
             None => return Ok(false),
         };
 
+        // Check queue-level limits
+        if let Some(qcfg) = self.queue_configs.get(&job.queue) {
+            if let Some(ref rl_config) = qcfg.rate_limit {
+                let key = format!("queue:{}", job.queue);
+                if !self.rate_limiter.try_acquire(&key, rl_config)? {
+                    self.storage
+                        .retry(&job.id, now + RATE_LIMIT_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+            if let Some(max_conc) = qcfg.max_concurrent {
+                let stats = self.storage.stats_by_queue(&job.queue)?;
+                if stats.running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
+        }
+
         // Check circuit breaker for this task
         if let Some(config) = self.task_configs.get(&job.task_name) {
             if config.circuit_breaker.is_some() && !self.circuit_breaker.allow(&job.task_name)? {
@@ -68,6 +91,16 @@ impl Scheduler {
                     return Ok(false);
                 }
             }
+
+            // Check per-task concurrency limit
+            if let Some(max_conc) = config.max_concurrent {
+                let running = self.storage.count_running_by_task(&job.task_name)?;
+                if running >= max_conc as i64 {
+                    self.storage
+                        .retry(&job.id, now + CONCURRENCY_RETRY_DELAY_MS)?;
+                    return Ok(false);
+                }
+            }
         }
 
         // Claim exactly-once execution

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/scheduler/result_handler.rs

@@ -3,11 +3,14 @@ use log::{error, warn};
 use crate::error::Result;
 use crate::storage::Storage;
 
-use super::{JobResult, Scheduler};
+use super::{JobResult, ResultOutcome, Scheduler};
 
 impl Scheduler {
     /// Handle a completed or failed job result from a worker.
-    pub fn handle_result(&self, result: JobResult) -> Result<()> {
+    ///
+    /// Returns a [`ResultOutcome`] describing the action taken, so the
+    /// caller can dispatch Python-side middleware hooks and events.
+    pub fn handle_result(&self, result: JobResult) -> Result<ResultOutcome> {
         match result {
             JobResult::Success {
                 job_id,
@@ -32,6 +35,11 @@ impl Scheduler {
                 if let Err(e) = self.circuit_breaker.record_success(task_name) {
                     error!("circuit breaker error for {task_name}: {e}");
                 }
+
+                Ok(ResultOutcome::Success {
+                    job_id,
+                    task_name: task_name.clone(),
+                })
             }
             JobResult::Failure {
                 job_id,
@@ -41,6 +49,7 @@ impl Scheduler {
                 task_name,
                 wall_time_ns,
                 should_retry,
+                timed_out,
             } => {
                 // Clear execution claim so it can be retried
                 if let Err(e) = self.storage.complete_execution(&job_id) {
@@ -68,7 +77,12 @@ impl Scheduler {
                         Some(job) => self.dlq.move_to_dlq(&job, &error, None)?,
                         None => warn!("job {job_id} disappeared before DLQ move"),
                     }
-                    return Ok(());
+                    return Ok(ResultOutcome::DeadLettered {
+                        job_id,
+                        task_name,
+                        error,
+                        timed_out,
+                    });
                 }
 
                 let policy = self
@@ -86,12 +100,25 @@ impl Scheduler {
                 if retry_count < effective_max {
                     let next_at = policy.next_retry_at(retry_count);
                     self.storage.retry(&job_id, next_at)?;
+                    Ok(ResultOutcome::Retry {
+                        job_id,
+                        task_name,
+                        error,
+                        retry_count,
+                        timed_out,
+                    })
                 } else {
                     // Move to DLQ
                     match self.storage.get_job(&job_id)? {
                         Some(job) => self.dlq.move_to_dlq(&job, &error, None)?,
                         None => warn!("job {job_id} disappeared before DLQ move"),
                     }
+                    Ok(ResultOutcome::DeadLettered {
+                        job_id,
+                        task_name,
+                        error,
+                        timed_out,
+                    })
                 }
             }
             JobResult::Cancelled {
@@ -113,8 +140,8 @@ impl Scheduler {
                 {
                     error!("failed to record metric for cancelled job {job_id}: {e}");
                 }
+                Ok(ResultOutcome::Cancelled { job_id, task_name })
             }
         }
-        Ok(())
     }
 }

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/diesel_common/jobs.rs

@@ -916,6 +916,19 @@ macro_rules! impl_diesel_job_ops {
                 Ok(affected as u64)
             }
 
+            /// Count running jobs for a specific task name (for per-task concurrency limiting).
+            pub fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+                let mut conn = self.conn()?;
+
+                let count: i64 = jobs::table
+                    .filter(jobs::task_name.eq(task_name))
+                    .filter(jobs::status.eq(JobStatus::Running as i32))
+                    .count()
+                    .get_result(&mut conn)?;
+
+                Ok(count)
+            }
+
             /// Purge job errors older than the given timestamp.
             pub fn purge_job_errors(&self, older_than_ms: i64) -> Result<u64> {
                 let mut conn = self.conn()?;

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/mod.rs

@@ -344,11 +344,18 @@ macro_rules! impl_storage {
                 worker_id: &str,
                 queues: &str,
                 tags: Option<&str>,
+                resources: Option<&str>,
+                resource_health: Option<&str>,
+                threads: i32,
             ) -> $crate::error::Result<()> {
-                self.register_worker(worker_id, queues, tags)
+                self.register_worker(worker_id, queues, tags, resources, resource_health, threads)
             }
-            fn heartbeat(&self, worker_id: &str) -> $crate::error::Result<()> {
-                self.heartbeat(worker_id)
+            fn heartbeat(
+                &self,
+                worker_id: &str,
+                resource_health: Option<&str>,
+            ) -> $crate::error::Result<()> {
+                self.heartbeat(worker_id, resource_health)
             }
             fn list_workers(
                 &self,
@@ -437,6 +444,12 @@ macro_rules! impl_storage {
             ) -> $crate::error::Result<u64> {
                 self.purge_execution_claims(older_than_ms)
             }
+            fn count_running_by_task(
+                &self,
+                task_name: &str,
+            ) -> $crate::error::Result<i64> {
+                self.count_running_by_task(task_name)
+            }
             fn stats_by_queue(
                 &self,
                 queue_name: &str,
@@ -705,11 +718,28 @@ impl Storage for StorageBackend {
     fn list_circuit_breakers(&self) -> Result<Vec<models::CircuitBreakerRow>> {
         delegate!(self, list_circuit_breakers)
     }
-    fn register_worker(&self, worker_id: &str, queues: &str, tags: Option<&str>) -> Result<()> {
-        delegate!(self, register_worker, worker_id, queues, tags)
+    fn register_worker(
+        &self,
+        worker_id: &str,
+        queues: &str,
+        tags: Option<&str>,
+        resources: Option<&str>,
+        resource_health: Option<&str>,
+        threads: i32,
+    ) -> Result<()> {
+        delegate!(
+            self,
+            register_worker,
+            worker_id,
+            queues,
+            tags,
+            resources,
+            resource_health,
+            threads
+        )
     }
-    fn heartbeat(&self, worker_id: &str) -> Result<()> {
-        delegate!(self, heartbeat, worker_id)
+    fn heartbeat(&self, worker_id: &str, resource_health: Option<&str>) -> Result<()> {
+        delegate!(self, heartbeat, worker_id, resource_health)
     }
     fn list_workers(&self) -> Result<Vec<models::WorkerRow>> {
         delegate!(self, list_workers)
@@ -768,6 +798,9 @@ impl Storage for StorageBackend {
     fn purge_execution_claims(&self, older_than_ms: i64) -> Result<u64> {
         delegate!(self, purge_execution_claims, older_than_ms)
     }
+    fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+        delegate!(self, count_running_by_task, task_name)
+    }
     fn stats_by_queue(&self, queue_name: &str) -> Result<QueueStats> {
         delegate!(self, stats_by_queue, queue_name)
     }

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/models.rs

@@ -284,6 +284,9 @@ pub struct WorkerRow {
     pub queues: String,
     pub status: String,
     pub tags: Option<String>,
+    pub resources: Option<String>,
+    pub resource_health: Option<String>,
+    pub threads: i32,
 }
 
 #[derive(Insertable, AsChangeset, Debug)]
@@ -294,6 +297,9 @@ pub struct NewWorkerRow<'a> {
     pub queues: &'a str,
     pub status: &'a str,
     pub tags: Option<&'a str>,
+    pub resources: Option<&'a str>,
+    pub resource_health: Option<&'a str>,
+    pub threads: i32,
 }
 
 // ── Queue State ─────────────────────────────────────────────────

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/postgres/mod.rs

@@ -340,6 +340,20 @@ impl PostgresStorage {
             "ALTER TABLE workers ADD COLUMN IF NOT EXISTS tags TEXT",
         );
 
+        // Migration: add resource advertisement columns to workers
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE workers ADD COLUMN IF NOT EXISTS resources TEXT",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE workers ADD COLUMN IF NOT EXISTS resource_health TEXT",
+        );
+        migration_alter(
+            &mut conn,
+            "ALTER TABLE workers ADD COLUMN IF NOT EXISTS threads INTEGER NOT NULL DEFAULT 0",
+        );
+
         diesel::sql_query(
             "CREATE TABLE IF NOT EXISTS queue_state (
                 queue_name TEXT PRIMARY KEY,

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/postgres/workers.rs

@@ -11,7 +11,15 @@ const DEAD_WORKER_THRESHOLD_MS: i64 = 30_000;
 
 impl PostgresStorage {
     /// Register a new worker or update an existing one.
-    pub fn register_worker(&self, worker_id: &str, queues: &str, tags: Option<&str>) -> Result<()> {
+    pub fn register_worker(
+        &self,
+        worker_id: &str,
+        queues: &str,
+        tags: Option<&str>,
+        resources: Option<&str>,
+        resource_health: Option<&str>,
+        threads: i32,
+    ) -> Result<()> {
         let mut conn = self.conn()?;
         let now = now_millis();
 
@@ -21,6 +29,9 @@ impl PostgresStorage {
             queues,
             status: "active",
             tags,
+            resources,
+            resource_health,
+            threads,
         };
 
         diesel::insert_into(workers::table)
@@ -34,13 +45,16 @@ impl PostgresStorage {
     }
 
     /// Update the heartbeat timestamp for a worker.
-    pub fn heartbeat(&self, worker_id: &str) -> Result<()> {
+    pub fn heartbeat(&self, worker_id: &str, resource_health: Option<&str>) -> Result<()> {
         let mut conn = self.conn()?;
         let now = now_millis();
 
         diesel::update(workers::table)
             .filter(workers::worker_id.eq(worker_id))
-            .set(workers::last_heartbeat.eq(now))
+            .set((
+                workers::last_heartbeat.eq(now),
+                workers::resource_health.eq(resource_health),
+            ))
             .execute(&mut conn)?;
 
         Ok(())

{taskito-0.4.0 → taskito-0.6.0}/crates/taskito-core/src/storage/redis_backend/jobs.rs

@@ -725,6 +725,24 @@ impl RedisStorage {
         Ok(stats)
     }
 
+    /// Count running jobs for a specific task name (for per-task concurrency limiting).
+    pub fn count_running_by_task(&self, task_name: &str) -> Result<i64> {
+        let mut conn = self.conn()?;
+        let by_task_key = self.key(&["jobs", "by_task", task_name]);
+        let job_ids: Vec<String> = conn.smembers(&by_task_key).map_err(map_err)?;
+
+        let mut count: i64 = 0;
+        for id in &job_ids {
+            if let Some(job) = self.load_job(&mut conn, id)? {
+                if job.status == JobStatus::Running {
+                    count += 1;
+                }
+            }
+        }
+
+        Ok(count)
+    }
+
     pub fn stats_by_queue(&self, queue_name: &str) -> Result<QueueStats> {
         let mut conn = self.conn()?;
         let by_queue_key = self.key(&["jobs", "by_queue", queue_name]);