quebec 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {quebec-0.3.0 → quebec-0.3.2}/Cargo.lock +1 -1
- {quebec-0.3.0 → quebec-0.3.2}/Cargo.toml +1 -1
- {quebec-0.3.0 → quebec-0.3.2}/PKG-INFO +1 -1
- {quebec-0.3.0 → quebec-0.3.2}/pyproject.toml +1 -1
- {quebec-0.3.0 → quebec-0.3.2}/src/config.rs +32 -1
- {quebec-0.3.0 → quebec-0.3.2}/src/context.rs +83 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/failed_jobs.rs +56 -1
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/health.rs +11 -3
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/job_details.rs +16 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/queues.rs +6 -33
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/recurring_jobs.rs +4 -3
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/workers.rs +9 -2
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/models.rs +46 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/base.html +31 -10
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/job-details.html +11 -1
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/queues.html +2 -2
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/stats.html +19 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/workers.html +3 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/utils.rs +41 -2
- {quebec-0.3.0 → quebec-0.3.2}/src/core.rs +13 -9
- {quebec-0.3.0 → quebec-0.3.2}/src/dispatcher.rs +16 -9
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_failed_executions.rs +64 -14
- {quebec-0.3.0 → quebec-0.3.2}/src/notify.rs +99 -10
- {quebec-0.3.0 → quebec-0.3.2}/src/process.rs +58 -5
- {quebec-0.3.0 → quebec-0.3.2}/src/query_builder.rs +125 -4
- {quebec-0.3.0 → quebec-0.3.2}/src/scheduler.rs +101 -8
- {quebec-0.3.0 → quebec-0.3.2}/src/semaphore.rs +8 -6
- {quebec-0.3.0 → quebec-0.3.2}/src/types.rs +821 -3
- {quebec-0.3.0 → quebec-0.3.2}/src/worker.rs +75 -28
- {quebec-0.3.0 → quebec-0.3.2}/.gitattributes +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/.gitignore +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/LICENSE +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/README.md +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/docs/images/control-plane.png +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/__init__.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/__main__.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/context.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/logger.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/sqlalchemy.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/structlog.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/python/quebec/supervisor.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/quickstart.py +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/continuation.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/ext.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/blocked_jobs.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/events.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/finished_jobs.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/in_progress_jobs.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/mod.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/overview.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/handlers/scheduled_jobs.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/mod.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/static/chart.min.js +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/static/stimulus.min.js +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/static/tailwindcss.js +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/static/turbo.es2017-esm.min.js +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/blocked-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/failed-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/finished-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/in-progress-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/overview.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/queue-details.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/recurring-jobs-schedule.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/recurring-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates/scheduled-jobs.html +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/control_plane/templates.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/database_url.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/mod.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/prelude.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_blocked_executions.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_claimed_executions.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_jobs.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_pauses.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_processes.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_ready_executions.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_recurring_executions.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_recurring_tasks.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_scheduled_executions.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/entities/quebec_semaphores.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/error.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/lib.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/proctitle_macos.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/proctitle_unix.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/schema_builder.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/supervisor.rs +0 -0
- {quebec-0.3.0 → quebec-0.3.2}/src/utils.rs +0 -0
|
@@ -340,7 +340,11 @@ impl QueueConfig {
|
|
|
340
340
|
/// Parse YAML string into configuration
|
|
341
341
|
pub fn parse_yaml(yaml_str: &str, env: Option<&str>) -> Result<Self> {
|
|
342
342
|
let yaml_str = Self::expand_env_vars(yaml_str);
|
|
343
|
-
let value: serde_yaml::Value = serde_yaml::from_str(&yaml_str)?;
|
|
343
|
+
let mut value: serde_yaml::Value = serde_yaml::from_str(&yaml_str)?;
|
|
344
|
+
// Expand YAML merge keys (`<<: *anchor`) so configs that factor a
|
|
345
|
+
// default block via anchors work. serde_yaml only honors merge keys
|
|
346
|
+
// when explicitly asked.
|
|
347
|
+
value.apply_merge()?;
|
|
344
348
|
|
|
345
349
|
match &value {
|
|
346
350
|
serde_yaml::Value::Mapping(map) => {
|
|
@@ -496,3 +500,30 @@ const DEFAULT_CONFIG_PATHS: &[&str] = &[
|
|
|
496
500
|
"queue.yml", // Current directory (Python projects)
|
|
497
501
|
"config/queue.yml", // Solid Queue compatible (Rails projects)
|
|
498
502
|
];
|
|
503
|
+
|
|
504
|
+
#[cfg(test)]
|
|
505
|
+
mod tests {
|
|
506
|
+
use super::*;
|
|
507
|
+
|
|
508
|
+
#[test]
|
|
509
|
+
fn parses_yaml_merge_keys_in_queue_config() {
|
|
510
|
+
let yaml = r#"
|
|
511
|
+
defaults: &defaults
|
|
512
|
+
workers:
|
|
513
|
+
- queues: "*"
|
|
514
|
+
threads: 3
|
|
515
|
+
polling_interval: 0.1
|
|
516
|
+
|
|
517
|
+
development:
|
|
518
|
+
<<: *defaults
|
|
519
|
+
|
|
520
|
+
production:
|
|
521
|
+
<<: *defaults
|
|
522
|
+
"#;
|
|
523
|
+
let dev = QueueConfig::parse_yaml(yaml, Some("development"))
|
|
524
|
+
.expect("merge key in queue config must parse");
|
|
525
|
+
let workers = dev.workers.as_ref().expect("workers populated by merge");
|
|
526
|
+
assert_eq!(workers.len(), 1);
|
|
527
|
+
assert_eq!(workers[0].threads, Some(3));
|
|
528
|
+
}
|
|
529
|
+
}
|
|
@@ -273,6 +273,25 @@ impl DiscardStrategy {
|
|
|
273
273
|
}
|
|
274
274
|
}
|
|
275
275
|
|
|
276
|
+
/// Replace characters that would break downstream consumers if they ended up
|
|
277
|
+
/// inside a queue name. Primary motivation is the control-plane router which
|
|
278
|
+
/// uses queue_name as an `:name` path segment (anything with `/` would 404),
|
|
279
|
+
/// but space / `?` / `#` / `%` / control chars are also URL-hostile and worth
|
|
280
|
+
/// neutralising. Returns the cleaned string and a flag indicating whether any
|
|
281
|
+
/// substitution happened so callers can warn the operator once.
|
|
282
|
+
pub(crate) fn sanitize_queue_name(raw: &str) -> (String, bool) {
|
|
283
|
+
let cleaned: String = raw
|
|
284
|
+
.chars()
|
|
285
|
+
.map(|c| match c {
|
|
286
|
+
'/' | '\\' | '?' | '#' | '%' => '-',
|
|
287
|
+
c if c.is_whitespace() || c.is_control() => '-',
|
|
288
|
+
c => c,
|
|
289
|
+
})
|
|
290
|
+
.collect();
|
|
291
|
+
let changed = cleaned != raw;
|
|
292
|
+
(cleaned, changed)
|
|
293
|
+
}
|
|
294
|
+
|
|
276
295
|
#[derive(Debug)]
|
|
277
296
|
pub struct AppContext {
|
|
278
297
|
pub cwd: std::path::PathBuf,
|
|
@@ -285,6 +304,21 @@ pub struct AppContext {
|
|
|
285
304
|
/// Disable when running through transaction-pooling proxies (RDS Proxy,
|
|
286
305
|
/// PgBouncer transaction mode) that don't keep session state across queries.
|
|
287
306
|
pub use_listen_notify: bool,
|
|
307
|
+
/// Per-queue minimum interval between NOTIFYs emitted by this process.
|
|
308
|
+
/// Producers that fall inside the window are silently dropped; the worker
|
|
309
|
+
/// catches up via polling / IDLE fallback. Set to `Duration::ZERO` to
|
|
310
|
+
/// disable throttling. Mitigates NOTIFY storms on Aurora-style clusters
|
|
311
|
+
/// where every NOTIFY incurs cross-AZ replication cost.
|
|
312
|
+
pub notify_throttle_interval: Duration,
|
|
313
|
+
/// When set, every enqueue path rewrites `queue_name` to this value,
|
|
314
|
+
/// ignoring whatever the class / call site / scheduler specified. Use
|
|
315
|
+
/// for multi-branch development against a shared database: each branch
|
|
316
|
+
/// runs with its own `QUEBEC_FORCE_OVERRIDE_QUEUE=branch_x` and only
|
|
317
|
+
/// consumes that queue, so jobs enqueued by one branch are never picked
|
|
318
|
+
/// up by another. Production deployments should leave this unset — the
|
|
319
|
+
/// silent rewrite is intentional for the dev use case but surprising
|
|
320
|
+
/// elsewhere.
|
|
321
|
+
pub force_override_queue: Option<String>,
|
|
288
322
|
pub process_heartbeat_interval: Duration,
|
|
289
323
|
pub process_alive_threshold: Duration,
|
|
290
324
|
pub shutdown_timeout: Duration,
|
|
@@ -452,6 +486,9 @@ impl AppContext {
|
|
|
452
486
|
if let Some(v) = get_bool("use_listen_notify") {
|
|
453
487
|
ctx.use_listen_notify = v;
|
|
454
488
|
}
|
|
489
|
+
if let Some(v) = get_duration("notify_throttle_interval") {
|
|
490
|
+
ctx.notify_throttle_interval = v;
|
|
491
|
+
}
|
|
455
492
|
if let Some(v) = get_duration("process_heartbeat_interval") {
|
|
456
493
|
ctx.process_heartbeat_interval = v;
|
|
457
494
|
}
|
|
@@ -527,6 +564,34 @@ impl AppContext {
|
|
|
527
564
|
if let Some(v) = get_u64("worker_threads") {
|
|
528
565
|
ctx.worker_threads = v;
|
|
529
566
|
}
|
|
567
|
+
// force_override_queue: kwargs win over the env var the Default
|
|
568
|
+
// already read. Empty / whitespace-only strings clear the override
|
|
569
|
+
// so callers can explicitly opt out from Python even when
|
|
570
|
+
// QUEBEC_FORCE_OVERRIDE_QUEUE is set in the environment.
|
|
571
|
+
if let Some(val) = options.get("force_override_queue") {
|
|
572
|
+
match val.extract::<String>(py) {
|
|
573
|
+
Ok(q) => {
|
|
574
|
+
let trimmed = q.trim();
|
|
575
|
+
ctx.force_override_queue = if trimmed.is_empty() {
|
|
576
|
+
None
|
|
577
|
+
} else {
|
|
578
|
+
let (cleaned, changed) = sanitize_queue_name(trimmed);
|
|
579
|
+
if changed {
|
|
580
|
+
warn!(
|
|
581
|
+
"force_override_queue='{}' sanitized to '{}' \
|
|
582
|
+
(URL-hostile characters replaced with '-')",
|
|
583
|
+
trimmed, cleaned
|
|
584
|
+
);
|
|
585
|
+
}
|
|
586
|
+
Some(cleaned)
|
|
587
|
+
};
|
|
588
|
+
}
|
|
589
|
+
Err(_) => warn!(
|
|
590
|
+
"Config 'force_override_queue': expected String, got {:?}",
|
|
591
|
+
val.bind(py).get_type()
|
|
592
|
+
),
|
|
593
|
+
}
|
|
594
|
+
}
|
|
530
595
|
// table_name_prefix: only apply if non-empty
|
|
531
596
|
if let Some(val) = options.get("table_name_prefix") {
|
|
532
597
|
match val.extract::<String>(py) {
|
|
@@ -572,6 +637,22 @@ impl AppContext {
|
|
|
572
637
|
name: std::env::var("QUEBEC_NAME").unwrap_or_else(|_| "quebec".to_string()),
|
|
573
638
|
use_skip_locked: true,
|
|
574
639
|
use_listen_notify: true,
|
|
640
|
+
notify_throttle_interval: Duration::from_secs(1),
|
|
641
|
+
force_override_queue: std::env::var("QUEBEC_FORCE_OVERRIDE_QUEUE")
|
|
642
|
+
.ok()
|
|
643
|
+
.map(|s| s.trim().to_string())
|
|
644
|
+
.filter(|s| !s.is_empty())
|
|
645
|
+
.map(|raw| {
|
|
646
|
+
let (cleaned, changed) = sanitize_queue_name(&raw);
|
|
647
|
+
if changed {
|
|
648
|
+
warn!(
|
|
649
|
+
"QUEBEC_FORCE_OVERRIDE_QUEUE='{}' sanitized to '{}' \
|
|
650
|
+
(URL-hostile characters replaced with '-')",
|
|
651
|
+
raw, cleaned
|
|
652
|
+
);
|
|
653
|
+
}
|
|
654
|
+
cleaned
|
|
655
|
+
}),
|
|
575
656
|
process_heartbeat_interval: Duration::from_secs(60),
|
|
576
657
|
process_alive_threshold: Duration::from_secs(300),
|
|
577
658
|
shutdown_timeout: Duration::from_secs(5),
|
|
@@ -688,6 +769,8 @@ impl AppContext {
|
|
|
688
769
|
// Fresh state; child must call `watch_parent_pid` after fork.
|
|
689
770
|
supervisor_pid: AtomicI32::new(0),
|
|
690
771
|
use_listen_notify: self.use_listen_notify,
|
|
772
|
+
notify_throttle_interval: self.notify_throttle_interval,
|
|
773
|
+
force_override_queue: self.force_override_queue.clone(),
|
|
691
774
|
}
|
|
692
775
|
}
|
|
693
776
|
|
|
@@ -88,12 +88,17 @@ impl ControlPlane {
|
|
|
88
88
|
// Get total count of failed jobs for pagination calculation
|
|
89
89
|
let start = Instant::now();
|
|
90
90
|
|
|
91
|
-
// Execute count query using query_builder (with same filters)
|
|
91
|
+
// Execute count query using query_builder (with same filters).
|
|
92
|
+
// The since/until/error_like filters are only used by the bulk
|
|
93
|
+
// retry/discard endpoints — the list view ignores them.
|
|
92
94
|
let total_count: u64 = query_builder::failed_executions::count_all(
|
|
93
95
|
db,
|
|
94
96
|
table_config,
|
|
95
97
|
pagination.class_name.as_deref(),
|
|
96
98
|
pagination.queue_name.as_deref(),
|
|
99
|
+
None,
|
|
100
|
+
None,
|
|
101
|
+
None,
|
|
97
102
|
)
|
|
98
103
|
.await
|
|
99
104
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
@@ -245,6 +250,11 @@ impl ControlPlane {
|
|
|
245
250
|
Query(pagination): Query<Pagination>,
|
|
246
251
|
headers: HeaderMap,
|
|
247
252
|
) -> Response {
|
|
253
|
+
let (since, until) = match parse_time_bounds(&pagination) {
|
|
254
|
+
Ok(v) => v,
|
|
255
|
+
Err(msg) => return (StatusCode::BAD_REQUEST, msg).into_response(),
|
|
256
|
+
};
|
|
257
|
+
|
|
248
258
|
let db = match state.ctx.get_db().await {
|
|
249
259
|
Ok(db) => db,
|
|
250
260
|
Err(_) => return Self::error_response(),
|
|
@@ -254,6 +264,7 @@ impl ControlPlane {
|
|
|
254
264
|
let redirect = Self::referer_without_page_or(&headers, "/failed-jobs");
|
|
255
265
|
let class_name = pagination.class_name.clone();
|
|
256
266
|
let queue_name = pagination.queue_name.clone();
|
|
267
|
+
let error_like = pagination.error_like.clone();
|
|
257
268
|
|
|
258
269
|
match db
|
|
259
270
|
.transaction::<_, u64, DbErr>(|txn| {
|
|
@@ -264,6 +275,9 @@ impl ControlPlane {
|
|
|
264
275
|
&table_config,
|
|
265
276
|
class_name.as_deref(),
|
|
266
277
|
queue_name.as_deref(),
|
|
278
|
+
since,
|
|
279
|
+
until,
|
|
280
|
+
error_like.as_deref(),
|
|
267
281
|
)
|
|
268
282
|
.await?;
|
|
269
283
|
Ok(count)
|
|
@@ -288,6 +302,11 @@ impl ControlPlane {
|
|
|
288
302
|
Query(pagination): Query<Pagination>,
|
|
289
303
|
headers: HeaderMap,
|
|
290
304
|
) -> Response {
|
|
305
|
+
let (since, until) = match parse_time_bounds(&pagination) {
|
|
306
|
+
Ok(v) => v,
|
|
307
|
+
Err(msg) => return (StatusCode::BAD_REQUEST, msg).into_response(),
|
|
308
|
+
};
|
|
309
|
+
|
|
291
310
|
let db = match state.ctx.get_db().await {
|
|
292
311
|
Ok(db) => db,
|
|
293
312
|
Err(_) => return Self::error_response(),
|
|
@@ -297,6 +316,7 @@ impl ControlPlane {
|
|
|
297
316
|
let redirect = Self::referer_without_page_or(&headers, "/failed-jobs");
|
|
298
317
|
let class_name = pagination.class_name.clone();
|
|
299
318
|
let queue_name = pagination.queue_name.clone();
|
|
319
|
+
let error_like = pagination.error_like.clone();
|
|
300
320
|
|
|
301
321
|
match db
|
|
302
322
|
.transaction::<_, u64, DbErr>(|txn| {
|
|
@@ -307,6 +327,9 @@ impl ControlPlane {
|
|
|
307
327
|
&table_config,
|
|
308
328
|
class_name.as_deref(),
|
|
309
329
|
queue_name.as_deref(),
|
|
330
|
+
since,
|
|
331
|
+
until,
|
|
332
|
+
error_like.as_deref(),
|
|
310
333
|
)
|
|
311
334
|
.await?;
|
|
312
335
|
Ok(count)
|
|
@@ -325,3 +348,35 @@ impl ControlPlane {
|
|
|
325
348
|
}
|
|
326
349
|
}
|
|
327
350
|
}
|
|
351
|
+
|
|
352
|
+
/// Parse the optional `since` / `until` query strings into `NaiveDateTime`.
|
|
353
|
+
/// Accepted formats: RFC 3339 with timezone (`2026-05-17T12:00:00Z`),
|
|
354
|
+
/// naive ISO (`2026-05-17T12:00:00`), or `YYYY-MM-DD HH:MM:SS`. RFC 3339
|
|
355
|
+
/// values are normalized to UTC before being stripped of the offset, matching
|
|
356
|
+
/// how `created_at` is stored in the DB.
|
|
357
|
+
fn parse_time_bounds(
|
|
358
|
+
pagination: &Pagination,
|
|
359
|
+
) -> Result<(Option<chrono::NaiveDateTime>, Option<chrono::NaiveDateTime>), String> {
|
|
360
|
+
let parse = |raw: &str| -> Result<chrono::NaiveDateTime, String> {
|
|
361
|
+
if let Ok(dt) = chrono::DateTime::parse_from_rfc3339(raw) {
|
|
362
|
+
return Ok(dt.naive_utc());
|
|
363
|
+
}
|
|
364
|
+
chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%dT%H:%M:%S")
|
|
365
|
+
.or_else(|_| chrono::NaiveDateTime::parse_from_str(raw, "%Y-%m-%d %H:%M:%S"))
|
|
366
|
+
.map_err(|e| format!("invalid timestamp '{raw}': {e}"))
|
|
367
|
+
};
|
|
368
|
+
|
|
369
|
+
let since = pagination
|
|
370
|
+
.since
|
|
371
|
+
.as_deref()
|
|
372
|
+
.map(parse)
|
|
373
|
+
.transpose()
|
|
374
|
+
.map_err(|e| format!("since: {e}"))?;
|
|
375
|
+
let until = pagination
|
|
376
|
+
.until
|
|
377
|
+
.as_deref()
|
|
378
|
+
.map(parse)
|
|
379
|
+
.transpose()
|
|
380
|
+
.map_err(|e| format!("until: {e}"))?;
|
|
381
|
+
Ok((since, until))
|
|
382
|
+
}
|
|
@@ -101,9 +101,17 @@ impl ControlPlane {
|
|
|
101
101
|
let blocked = query_builder::blocked_executions::count_all(db, table_config, None, None)
|
|
102
102
|
.await
|
|
103
103
|
.map_err(map_err)?;
|
|
104
|
-
let failed = query_builder::failed_executions::count_all(
|
|
105
|
-
|
|
106
|
-
|
|
104
|
+
let failed = query_builder::failed_executions::count_all(
|
|
105
|
+
db,
|
|
106
|
+
table_config,
|
|
107
|
+
None,
|
|
108
|
+
None,
|
|
109
|
+
None,
|
|
110
|
+
None,
|
|
111
|
+
None,
|
|
112
|
+
)
|
|
113
|
+
.await
|
|
114
|
+
.map_err(map_err)?;
|
|
107
115
|
|
|
108
116
|
// Status determination
|
|
109
117
|
// "unavailable" only when workers exist but all are dead, or no processes at all
|
|
@@ -343,8 +343,24 @@ impl ControlPlane {
|
|
|
343
343
|
// Format arguments for display
|
|
344
344
|
job_details.arguments = Self::parse_arguments(&job_details.arguments);
|
|
345
345
|
|
|
346
|
+
// Surface arguments.exception_executions (class → retry count) so the
|
|
347
|
+
// template can list it. ActiveJob doesn't persist message/backtrace
|
|
348
|
+
// for retried exceptions; this map is the only signal that survives.
|
|
349
|
+
let exception_executions: Option<serde_json::Map<String, serde_json::Value>> =
|
|
350
|
+
serde_json::from_str::<serde_json::Value>(&job_details.arguments)
|
|
351
|
+
.ok()
|
|
352
|
+
.and_then(|v| {
|
|
353
|
+
v.get("exception_executions")
|
|
354
|
+
.and_then(|m| m.as_object())
|
|
355
|
+
.cloned()
|
|
356
|
+
})
|
|
357
|
+
.filter(|m| !m.is_empty());
|
|
358
|
+
|
|
346
359
|
let mut context = tera::Context::new();
|
|
347
360
|
context.insert("job", &job_details);
|
|
361
|
+
if let Some(map) = &exception_executions {
|
|
362
|
+
context.insert("exception_executions", map);
|
|
363
|
+
}
|
|
348
364
|
context.insert(
|
|
349
365
|
"active_page",
|
|
350
366
|
match status.as_str() {
|
|
@@ -13,7 +13,7 @@ use std::time::Instant;
|
|
|
13
13
|
use tracing::{debug, error};
|
|
14
14
|
|
|
15
15
|
use crate::control_plane::{
|
|
16
|
-
models::{Pagination, QueueInfo},
|
|
16
|
+
models::{queue_slug, Pagination, QueueInfo},
|
|
17
17
|
ControlPlane,
|
|
18
18
|
};
|
|
19
19
|
use crate::query_builder;
|
|
@@ -42,39 +42,11 @@ impl ControlPlane {
|
|
|
42
42
|
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
43
43
|
|
|
44
44
|
let table_config = &state.ctx.table_config;
|
|
45
|
-
let ready_table = Alias::new(&table_config.ready_executions);
|
|
46
45
|
|
|
47
46
|
// Count only ready executions per queue (matching Solid Queue's Queue#size)
|
|
48
|
-
let
|
|
49
|
-
.column(Alias::new("queue_name"))
|
|
50
|
-
.expr_as(
|
|
51
|
-
Expr::col(Alias::new("queue_name")).count(),
|
|
52
|
-
Alias::new("count"),
|
|
53
|
-
)
|
|
54
|
-
.from(ready_table)
|
|
55
|
-
.group_by_col(Alias::new("queue_name"))
|
|
56
|
-
.to_owned();
|
|
57
|
-
|
|
58
|
-
let (sql, values) = match db.get_database_backend() {
|
|
59
|
-
DbBackend::Postgres => query.build(PostgresQueryBuilder),
|
|
60
|
-
DbBackend::Sqlite => query.build(SqliteQueryBuilder),
|
|
61
|
-
DbBackend::MySql => query.build(MysqlQueryBuilder),
|
|
62
|
-
};
|
|
63
|
-
|
|
64
|
-
let stmt = Statement::from_sql_and_values(db.get_database_backend(), &sql, values);
|
|
65
|
-
|
|
66
|
-
// Get queue counts with unfinished jobs
|
|
67
|
-
let queue_counts_map: HashMap<String, i64> = db
|
|
68
|
-
.query_all(stmt)
|
|
47
|
+
let queue_counts_map = query_builder::ready_executions::count_by_queue(db, table_config)
|
|
69
48
|
.await
|
|
70
|
-
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))
|
|
71
|
-
.into_iter()
|
|
72
|
-
.map(|row| {
|
|
73
|
-
let queue_name: String = row.try_get("", "queue_name").unwrap_or_default();
|
|
74
|
-
let count: i64 = row.try_get("", "count").unwrap_or_default();
|
|
75
|
-
(queue_name, count)
|
|
76
|
-
})
|
|
77
|
-
.collect();
|
|
49
|
+
.map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e.to_string()))?;
|
|
78
50
|
|
|
79
51
|
// Ensure all queues are included in results, even if they have no unfinished jobs
|
|
80
52
|
let queue_counts: Vec<(String, i64)> = all_queue_names
|
|
@@ -102,13 +74,14 @@ impl ControlPlane {
|
|
|
102
74
|
let queue_infos: Vec<QueueInfo> = queue_counts
|
|
103
75
|
.into_iter()
|
|
104
76
|
.map(|(name, count)| QueueInfo {
|
|
105
|
-
|
|
106
|
-
jobs_count: count,
|
|
77
|
+
slug: queue_slug(&name),
|
|
107
78
|
status: if paused_queue_names.contains(&name) {
|
|
108
79
|
"paused".to_string()
|
|
109
80
|
} else {
|
|
110
81
|
"active".to_string()
|
|
111
82
|
},
|
|
83
|
+
name,
|
|
84
|
+
jobs_count: count,
|
|
112
85
|
})
|
|
113
86
|
.filter(|queue| {
|
|
114
87
|
// Apply status filter if provided
|
|
@@ -152,10 +152,11 @@ impl ControlPlane {
|
|
|
152
152
|
})
|
|
153
153
|
.await?;
|
|
154
154
|
|
|
155
|
-
// Send NOTIFY after transaction commits (only if job was created)
|
|
155
|
+
// Send NOTIFY after transaction commits (only if job was created).
|
|
156
|
+
// `should_send_notify` enforces backend + use_listen_notify + per-queue throttle.
|
|
156
157
|
if let Some(ref queue) = enqueued_queue {
|
|
157
|
-
if state.ctx
|
|
158
|
-
NotifyManager::send_notify(&state.ctx.name, db, queue
|
|
158
|
+
if crate::notify::should_send_notify(&state.ctx, queue) {
|
|
159
|
+
NotifyManager::send_notify(&state.ctx.name, db, queue)
|
|
159
160
|
.await
|
|
160
161
|
.inspect_err(|e| warn!("Failed to send NOTIFY: {}", e))
|
|
161
162
|
.ok();
|
|
@@ -47,12 +47,18 @@ impl ControlPlane {
|
|
|
47
47
|
"alive"
|
|
48
48
|
};
|
|
49
49
|
|
|
50
|
-
let
|
|
50
|
+
let metadata = worker
|
|
51
51
|
.metadata
|
|
52
52
|
.as_deref()
|
|
53
|
-
.and_then(|m| serde_json::from_str::<serde_json::Value>(m).ok())
|
|
53
|
+
.and_then(|m| serde_json::from_str::<serde_json::Value>(m).ok());
|
|
54
|
+
let quiet = metadata
|
|
55
|
+
.as_ref()
|
|
54
56
|
.and_then(|v| v.get("quiet").and_then(|q| q.as_bool()))
|
|
55
57
|
.unwrap_or(false);
|
|
58
|
+
let revision = metadata
|
|
59
|
+
.as_ref()
|
|
60
|
+
.and_then(|v| v.get("revision").and_then(|s| s.as_str()))
|
|
61
|
+
.map(|s| s.to_string());
|
|
56
62
|
|
|
57
63
|
WorkerInfo {
|
|
58
64
|
id: worker.id,
|
|
@@ -64,6 +70,7 @@ impl ControlPlane {
|
|
|
64
70
|
seconds_since_heartbeat,
|
|
65
71
|
status: status.to_string(),
|
|
66
72
|
quiet,
|
|
73
|
+
revision,
|
|
67
74
|
}
|
|
68
75
|
})
|
|
69
76
|
.collect();
|
|
@@ -3,10 +3,44 @@ use serde::{Deserialize, Serialize};
|
|
|
3
3
|
#[derive(Debug, Serialize)]
|
|
4
4
|
pub struct QueueInfo {
|
|
5
5
|
pub name: String,
|
|
6
|
+
/// HTML-id-safe version of `name` used as the anchor for turbo-stream
|
|
7
|
+
/// updates. Non `[A-Za-z0-9_-]` characters are replaced with `_`.
|
|
8
|
+
pub slug: String,
|
|
6
9
|
pub jobs_count: i64,
|
|
7
10
|
pub status: String,
|
|
8
11
|
}
|
|
9
12
|
|
|
13
|
+
/// Encode a queue name into an HTML-id-safe slug with **no collisions**.
|
|
14
|
+
/// Used to anchor per-queue turbo-stream targets (`queue-count-{slug}`,
|
|
15
|
+
/// `queue-status-{slug}`).
|
|
16
|
+
///
|
|
17
|
+
/// Standard URL percent-encoding semantics, byte-by-byte: `[A-Za-z0-9_-]`
|
|
18
|
+
/// pass through unchanged, everything else becomes `%HH` (uppercase hex of
|
|
19
|
+
/// the UTF-8 byte). Guarantees a one-to-one mapping — a naive "replace
|
|
20
|
+
/// with `_`" strategy would collapse `a.b`, `a/b`, `a b` onto the same DOM
|
|
21
|
+
/// element and silently freeze all but the first one's live counter.
|
|
22
|
+
///
|
|
23
|
+
/// `%` in an HTML `id` attribute is legal; turbo-stream `target=` looks the
|
|
24
|
+
/// element up via `document.getElementById`, which doesn't need CSS-style
|
|
25
|
+
/// escaping.
|
|
26
|
+
///
|
|
27
|
+
/// Examples:
|
|
28
|
+
/// * `"auto"` → `"auto"`
|
|
29
|
+
/// * `"a.b"` → `"a%2Eb"`
|
|
30
|
+
/// * `"a/b"` → `"a%2Fb"`
|
|
31
|
+
/// * `"中"` → `"%E4%B8%AD"`
|
|
32
|
+
pub fn queue_slug(name: &str) -> String {
|
|
33
|
+
let mut out = String::with_capacity(name.len());
|
|
34
|
+
for b in name.bytes() {
|
|
35
|
+
if b.is_ascii_alphanumeric() || b == b'-' || b == b'_' {
|
|
36
|
+
out.push(b as char);
|
|
37
|
+
} else {
|
|
38
|
+
out.push_str(&format!("%{:02X}", b));
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
out
|
|
42
|
+
}
|
|
43
|
+
|
|
10
44
|
#[derive(Debug, Serialize)]
|
|
11
45
|
pub struct WorkerInfo {
|
|
12
46
|
pub id: i64,
|
|
@@ -18,6 +52,7 @@ pub struct WorkerInfo {
|
|
|
18
52
|
pub seconds_since_heartbeat: i64,
|
|
19
53
|
pub status: String,
|
|
20
54
|
pub quiet: bool,
|
|
55
|
+
pub revision: Option<String>,
|
|
21
56
|
}
|
|
22
57
|
|
|
23
58
|
#[derive(Debug, Deserialize)]
|
|
@@ -27,6 +62,17 @@ pub struct Pagination {
|
|
|
27
62
|
pub class_name: Option<String>,
|
|
28
63
|
pub queue_name: Option<String>,
|
|
29
64
|
pub status: Option<String>,
|
|
65
|
+
/// Lower bound on `failed_executions.created_at`. Accepted as a
|
|
66
|
+
/// chrono-parseable ISO 8601 / RFC 3339-ish string; handlers reject
|
|
67
|
+
/// unparseable input with 400. Currently only honoured by the
|
|
68
|
+
/// /failed-jobs/all/retry and /failed-jobs/all/delete endpoints.
|
|
69
|
+
pub since: Option<String>,
|
|
70
|
+
/// Upper bound on `failed_executions.created_at`. See `since`.
|
|
71
|
+
pub until: Option<String>,
|
|
72
|
+
/// SQL `LIKE` pattern matched against the stored error message
|
|
73
|
+
/// (e.g. `Process+crashed%`). Currently only honoured by the
|
|
74
|
+
/// /failed-jobs/all/retry and /failed-jobs/all/delete endpoints.
|
|
75
|
+
pub error_like: Option<String>,
|
|
30
76
|
}
|
|
31
77
|
|
|
32
78
|
fn default_page() -> u64 {
|
|
@@ -529,14 +529,21 @@
|
|
|
529
529
|
this.fetchSnapshot()
|
|
530
530
|
this.startPolling()
|
|
531
531
|
} else {
|
|
532
|
-
// SSE's first tick fires immediately on connect —
|
|
533
|
-
//
|
|
534
|
-
//
|
|
532
|
+
// SSE's first tick fires immediately on connect — but proxies or
|
|
533
|
+
// a slow first DB roundtrip can still delay it. Pull a snapshot
|
|
534
|
+
// now and let SSE overwrite once it catches up.
|
|
535
|
+
this.fetchSnapshot()
|
|
535
536
|
this.startSSE()
|
|
536
537
|
}
|
|
537
538
|
}
|
|
538
539
|
document.addEventListener("visibilitychange", this._onVisibility)
|
|
539
|
-
if (!document.hidden)
|
|
540
|
+
if (!document.hidden) {
|
|
541
|
+
// Eager snapshot on first paint: even though the page is SSR'd
|
|
542
|
+
// with current counts, the data may have drifted between server
|
|
543
|
+
// render and browser parse. SSE/polling will refresh again shortly.
|
|
544
|
+
this.fetchSnapshot()
|
|
545
|
+
this.startSSE()
|
|
546
|
+
}
|
|
540
547
|
}
|
|
541
548
|
|
|
542
549
|
async fetchSnapshot() {
|
|
@@ -564,6 +571,9 @@
|
|
|
564
571
|
if (!received) {
|
|
565
572
|
this.stopSSE()
|
|
566
573
|
this._usePolling = true
|
|
574
|
+
// setInterval waits a full interval before its first callback,
|
|
575
|
+
// so fetch immediately to avoid a multi-second blank window.
|
|
576
|
+
this.fetchSnapshot()
|
|
567
577
|
this.startPolling()
|
|
568
578
|
}
|
|
569
579
|
}
|
|
@@ -595,7 +605,18 @@
|
|
|
595
605
|
</head>
|
|
596
606
|
<body class="bg-white text-black min-h-screen flex flex-col">
|
|
597
607
|
<div class="max-w-screen-2xl mx-auto p-4 md:p-6 w-full flex-grow flex flex-col">
|
|
598
|
-
<
|
|
608
|
+
<div class="flex items-center gap-3 mb-6">
|
|
609
|
+
<h1 class="text-3xl font-bold">Control Plane</h1>
|
|
610
|
+
{% if force_override_queue %}
|
|
611
|
+
<span title="QUEBEC_FORCE_OVERRIDE_QUEUE is active — every enqueue is rewritten to this queue, ignoring class queue config and any `.set(queue=...)` call. Intended for multi-branch dev isolation; do not leave this set in production."
|
|
612
|
+
class="inline-flex items-center gap-1.5 rounded-md border border-amber-300 bg-amber-50 px-2.5 py-1 text-xs font-medium text-amber-900">
|
|
613
|
+
<svg xmlns="http://www.w3.org/2000/svg" class="h-3.5 w-3.5" viewBox="0 0 20 20" fill="currentColor" aria-hidden="true">
|
|
614
|
+
<path fill-rule="evenodd" d="M8.485 2.495c.673-1.167 2.357-1.167 3.03 0l6.28 10.875c.673 1.167-.17 2.625-1.516 2.625H3.72c-1.347 0-2.189-1.458-1.515-2.625L8.485 2.495zM10 5a.75.75 0 0 1 .75.75v4.5a.75.75 0 0 1-1.5 0v-4.5A.75.75 0 0 1 10 5zm0 9a1 1 0 1 0 0-2 1 1 0 0 0 0 2z" clip-rule="evenodd"/>
|
|
615
|
+
</svg>
|
|
616
|
+
Forced queue: <span class="font-mono">{{ force_override_queue }}</span>
|
|
617
|
+
</span>
|
|
618
|
+
{% endif %}
|
|
619
|
+
</div>
|
|
599
620
|
|
|
600
621
|
<div data-controller="nav" data-nav-active-class-value="border-b-2 border-black" class="flex flex-col flex-grow">
|
|
601
622
|
<turbo-frame id="nav-frame">
|
|
@@ -608,22 +629,22 @@
|
|
|
608
629
|
</a>
|
|
609
630
|
|
|
610
631
|
<a href="{{ base_path }}/failed-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
611
|
-
Failed jobs <span id="failed-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">0</span>
|
|
632
|
+
Failed jobs <span id="failed-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">{{ nav_failed_jobs | default(value=0) }}</span>
|
|
612
633
|
</a>
|
|
613
634
|
<a href="{{ base_path }}/in-progress-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
614
|
-
In Progress jobs <span id="in-progress-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">0</span>
|
|
635
|
+
In Progress jobs <span id="in-progress-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">{{ nav_in_progress_jobs | default(value=0) }}</span>
|
|
615
636
|
</a>
|
|
616
637
|
<a href="{{ base_path }}/blocked-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
617
|
-
Blocked jobs <span id="blocked-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">0</span>
|
|
638
|
+
Blocked jobs <span id="blocked-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">{{ nav_blocked_jobs | default(value=0) }}</span>
|
|
618
639
|
</a>
|
|
619
640
|
<a href="{{ base_path }}/scheduled-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
620
|
-
Scheduled jobs <span id="scheduled-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">0</span>
|
|
641
|
+
Scheduled jobs <span id="scheduled-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">{{ nav_scheduled_jobs | default(value=0) }}</span>
|
|
621
642
|
</a>
|
|
622
643
|
<a href="{{ base_path }}/recurring-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
623
644
|
Recurring jobs
|
|
624
645
|
</a>
|
|
625
646
|
<a href="{{ base_path }}/finished-jobs" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
626
|
-
Finished jobs <span id="finished-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">0</span>
|
|
647
|
+
Finished jobs <span id="finished-jobs-count" class="px-1 py-0.5 text-xs rounded-full bg-gray-200 text-gray-800">{{ finished_jobs_count | default(value=0) }}</span>
|
|
627
648
|
</a>
|
|
628
649
|
|
|
629
650
|
<a href="{{ base_path }}/workers" data-nav-target="navItem" data-action="click->nav#navigate" class="shrink-0 whitespace-nowrap px-4 h-8 font-medium text-gray-600 hover:text-black border-b-2 border-transparent hover:border-black">
|
|
@@ -197,9 +197,19 @@
|
|
|
197
197
|
{% endif %}
|
|
198
198
|
</div>
|
|
199
199
|
{% endfor %}
|
|
200
|
-
{%
|
|
200
|
+
{% elif not exception_executions %}
|
|
201
201
|
<div class="p-4 text-sm text-gray-500">No execution history available.</div>
|
|
202
202
|
{% endif %}
|
|
203
|
+
{% if exception_executions %}
|
|
204
|
+
<div class="p-4">
|
|
205
|
+
<div class="text-xs font-medium text-gray-700 mb-2">exception_executions</div>
|
|
206
|
+
<div class="space-y-1 text-sm">
|
|
207
|
+
{% for class_name, count in exception_executions %}
|
|
208
|
+
<div><span class="font-mono text-gray-900">{{ class_name }}</span>: {{ count }}</div>
|
|
209
|
+
{% endfor %}
|
|
210
|
+
</div>
|
|
211
|
+
</div>
|
|
212
|
+
{% endif %}
|
|
203
213
|
</div>
|
|
204
214
|
</div>
|
|
205
215
|
</div>
|