queuerPy 0.13.0__tar.gz → 0.15.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {queuerpy-0.13.0/queuerPy.egg-info → queuerpy-0.15.0}/PKG-INFO +1 -1
- {queuerpy-0.13.0 → queuerpy-0.15.0}/_version.py +1 -1
- {queuerpy-0.13.0 → queuerpy-0.15.0}/database/db_listener.py +0 -4
- {queuerpy-0.13.0 → queuerpy-0.15.0}/pyproject.toml +2 -2
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer.py +17 -7
- {queuerpy-0.13.0 → queuerpy-0.15.0/queuerPy.egg-info}/PKG-INFO +1 -1
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_job.py +3 -4
- {queuerpy-0.13.0 → queuerpy-0.15.0}/sql/job.sql +3 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/LICENSE +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/MANIFEST.in +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/README.md +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/__init__.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/__init__.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/broadcaster.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/listener.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/retryer.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/runner.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/scheduler.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/core/ticker.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/database/__init__.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/database/db_job.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/database/db_master.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/database/db_worker.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/__init__.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/database.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/error.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/logging.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/sql.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/helper/task.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/__init__.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/batch_job.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/connection.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/job.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/master.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/options.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/options_on_error.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/task.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/model/worker.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/py.typed +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuerPy.egg-info/SOURCES.txt +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuerPy.egg-info/dependency_links.txt +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuerPy.egg-info/requires.txt +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuerPy.egg-info/top_level.txt +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_global.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_listener.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_master.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_next_interval.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/queuer_task.py +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/setup.cfg +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/sql/master.sql +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/sql/notify.sql +0 -0
- {queuerpy-0.13.0 → queuerpy-0.15.0}/sql/worker.sql +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queuerPy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
|
|
5
5
|
Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
6
6
|
Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
@@ -97,10 +97,6 @@ class QueuerListener:
|
|
|
97
97
|
gen.__anext__(), timeout=0.1
|
|
98
98
|
)
|
|
99
99
|
if notify.channel == self.channel:
|
|
100
|
-
self.logger.debug(
|
|
101
|
-
f"Received notification: {notify.payload}"
|
|
102
|
-
)
|
|
103
|
-
# Process notification
|
|
104
100
|
try:
|
|
105
101
|
await notify_function(notify.payload or "")
|
|
106
102
|
notifications_processed = True
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "queuerPy"
|
|
7
|
-
version = "0.
|
|
7
|
+
version = "0.15.0"
|
|
8
8
|
description = "A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend"
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "Apache-2.0"
|
|
@@ -92,7 +92,7 @@ pythonVersion = "3.8"
|
|
|
92
92
|
pythonPlatform = "All"
|
|
93
93
|
|
|
94
94
|
[tool.mypy]
|
|
95
|
-
python_version = "0.
|
|
95
|
+
python_version = "0.15.0"
|
|
96
96
|
warn_return_any = true
|
|
97
97
|
warn_unused_configs = true
|
|
98
98
|
disallow_untyped_defs = true
|
|
@@ -169,10 +169,14 @@ class Queuer(
|
|
|
169
169
|
|
|
170
170
|
# Set up database listeners
|
|
171
171
|
try:
|
|
172
|
-
self.job_db_listener = new_queuer_db_listener(
|
|
172
|
+
self.job_db_listener = new_queuer_db_listener(
|
|
173
|
+
self.db_config,
|
|
174
|
+
"job",
|
|
175
|
+
)
|
|
173
176
|
logger.info("Added listener for channel: job")
|
|
174
177
|
self.job_archive_db_listener = new_queuer_db_listener(
|
|
175
|
-
self.db_config,
|
|
178
|
+
self.db_config,
|
|
179
|
+
"job_archive",
|
|
176
180
|
)
|
|
177
181
|
logger.info("Added listener for channel: job_archive")
|
|
178
182
|
except Exception as e:
|
|
@@ -413,7 +417,7 @@ class Queuer(
|
|
|
413
417
|
try:
|
|
414
418
|
job = Job.from_dict(json.loads(notification))
|
|
415
419
|
|
|
416
|
-
logger.debug(f"
|
|
420
|
+
logger.debug(f"Job added: {job.rid}")
|
|
417
421
|
|
|
418
422
|
if job.status in [JobStatus.QUEUED, JobStatus.SCHEDULED]:
|
|
419
423
|
go_func(self._run_job_initial, use_mp=False)
|
|
@@ -446,7 +450,6 @@ class Queuer(
|
|
|
446
450
|
and self.job_delete_broadcaster
|
|
447
451
|
):
|
|
448
452
|
await self.job_delete_broadcaster.broadcast(job)
|
|
449
|
-
logger.info(f"Broadcasted job deletion completion for {job.rid}")
|
|
450
453
|
|
|
451
454
|
except Exception as e:
|
|
452
455
|
logger.error(f"Error handling job archive notification: {e}")
|
|
@@ -523,15 +526,22 @@ class Queuer(
|
|
|
523
526
|
def _heartbeat_func(self) -> None:
|
|
524
527
|
"""Send periodic heartbeats - only updates database, not queuer state."""
|
|
525
528
|
try:
|
|
526
|
-
# Get current worker
|
|
527
|
-
|
|
529
|
+
# Get current worker with mutex
|
|
530
|
+
with self.worker_mutex:
|
|
531
|
+
current_worker = self.worker
|
|
532
|
+
|
|
533
|
+
updated_worker: Optional[Worker] = None
|
|
528
534
|
if current_worker:
|
|
529
535
|
# Update timestamp and save to database
|
|
530
536
|
current_worker.updated_at = datetime.now()
|
|
531
|
-
self.db_worker.update_worker(current_worker)
|
|
537
|
+
updated_worker = self.db_worker.update_worker(current_worker)
|
|
532
538
|
logger.debug(
|
|
533
539
|
f"Updated worker heartbeat timestamp: {current_worker.updated_at}"
|
|
534
540
|
)
|
|
541
|
+
|
|
542
|
+
if updated_worker:
|
|
543
|
+
with self.worker_mutex:
|
|
544
|
+
self.worker = updated_worker
|
|
535
545
|
except Exception as e:
|
|
536
546
|
logger.error(f"Heartbeat error: {e}")
|
|
537
547
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: queuerPy
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.15.0
|
|
4
4
|
Summary: A Python implementation of the queuer system - a job queuing and processing system with PostgreSQL backend
|
|
5
5
|
Author-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
6
6
|
Maintainer-email: Simon Herrmann <siherrmann@users.noreply.github.com>
|
|
@@ -352,8 +352,9 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
352
352
|
|
|
353
353
|
worker: Worker = getattr(self, "worker")
|
|
354
354
|
|
|
355
|
-
logger.debug(
|
|
356
|
-
|
|
355
|
+
logger.debug(
|
|
356
|
+
f"Worker available_tasks: {worker.available_tasks}, max_concurrency: {worker.max_concurrency}"
|
|
357
|
+
)
|
|
357
358
|
|
|
358
359
|
jobs = self.db_job.update_jobs_initial(worker)
|
|
359
360
|
if not jobs:
|
|
@@ -363,7 +364,6 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
363
364
|
logger.info(f"Found {len(jobs)} jobs to run")
|
|
364
365
|
|
|
365
366
|
for job in jobs:
|
|
366
|
-
logger.info(f"Processing job: {job.rid}")
|
|
367
367
|
if (
|
|
368
368
|
job.options
|
|
369
369
|
and job.options.schedule
|
|
@@ -382,7 +382,6 @@ class QueuerJobMixin(QueuerGlobalMixin):
|
|
|
382
382
|
|
|
383
383
|
logger.info(f"Scheduled job {job.rid} for {job.options.schedule.start}")
|
|
384
384
|
else:
|
|
385
|
-
logger.info(f"Running job immediately: {job.rid}")
|
|
386
385
|
runner = SmallRunner(self._run_job, job)
|
|
387
386
|
runner.go()
|
|
388
387
|
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|