celery-beat-sqlalchemy 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- celery_beat_sqlalchemy-0.0.1/.gitignore +8 -0
- celery_beat_sqlalchemy-0.0.1/LICENSE +19 -0
- celery_beat_sqlalchemy-0.0.1/PKG-INFO +13 -0
- celery_beat_sqlalchemy-0.0.1/README.md +0 -0
- celery_beat_sqlalchemy-0.0.1/pyproject.toml +25 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/__init__.py +0 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/db.py +18 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/models/__init__.py +0 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/models/base.py +3 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/models/celery_tasks.py +79 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/models/celery_tasks_schedule.py +175 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/models/celery_tasks_schedule_meta.py +46 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/scheduler.py +221 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/schemas/__init__.py +0 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/schemas/db/__init__.py +0 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/schemas/db/base.py +18 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/schemas/db/celery_tasks.py +11 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/schemas/db/celery_tasks_schedule.py +14 -0
- celery_beat_sqlalchemy-0.0.1/src/celery_beat_sqlalchemy/utils.py +26 -0
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Copyright (c) 2026 sqlalchemy_celery_beat
|
|
2
|
+
|
|
3
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
4
|
+
of this software and associated documentation files (the «Software»), to deal
|
|
5
|
+
in the Software without restriction, including without limitation the rights
|
|
6
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
7
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
8
|
+
furnished to do so, subject to the following conditions:
|
|
9
|
+
|
|
10
|
+
The above copyright notice and this permission notice shall be included in all
|
|
11
|
+
copies or substantial portions of the Software.
|
|
12
|
+
|
|
13
|
+
THE SOFTWARE IS PROVIDED «AS IS», WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
14
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
15
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
16
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
17
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
18
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
19
|
+
SOFTWARE.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: celery-beat-sqlalchemy
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Library for managing celery beat schedule through sqlalchemy models
|
|
5
|
+
Project-URL: Homepage, https://github.com/ebergard/sqlalchemy_celery_beat
|
|
6
|
+
Author-email: Kseniia Ebergard <ebergard-xu@bk.ru>
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
9
|
+
Classifier: Operating System :: OS Independent
|
|
10
|
+
Classifier: Programming Language :: Python :: 3
|
|
11
|
+
Requires-Python: >=3.8.0
|
|
12
|
+
Requires-Dist: celery>=5.4.0
|
|
13
|
+
Requires-Dist: sqlalchemy>=2.0.0
|
|
File without changes
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "celery-beat-sqlalchemy"
|
|
7
|
+
version = "0.0.1"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name="Kseniia Ebergard", email="ebergard-xu@bk.ru" },
|
|
10
|
+
]
|
|
11
|
+
description = "Library for managing celery beat schedule through sqlalchemy models"
|
|
12
|
+
readme = "README.md"
|
|
13
|
+
requires-python = ">=3.8.0"
|
|
14
|
+
dependencies = [
|
|
15
|
+
"celery>=5.4.0",
|
|
16
|
+
"sqlalchemy>=2.0.0"
|
|
17
|
+
]
|
|
18
|
+
classifiers = [
|
|
19
|
+
"Programming Language :: Python :: 3",
|
|
20
|
+
"License :: OSI Approved :: MIT License",
|
|
21
|
+
"Operating System :: OS Independent",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Homepage = "https://github.com/ebergard/sqlalchemy_celery_beat"
|
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import contextlib
|
|
2
|
+
from collections.abc import Generator
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from celery import current_app as celery_app
|
|
6
|
+
from sqlalchemy import NullPool, create_engine
|
|
7
|
+
from sqlalchemy.orm import Session
|
|
8
|
+
|
|
9
|
+
engine = create_engine(url=celery_app.conf.beat_db_scheduler_dsn, future=True, poolclass=NullPool)
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@contextlib.contextmanager
|
|
13
|
+
def db_sessionmaker() -> Generator[Session, Any, None]:
|
|
14
|
+
with (
|
|
15
|
+
engine.connect() as conn,
|
|
16
|
+
Session(bind=conn, expire_on_commit=False, autoflush=False) as session,
|
|
17
|
+
):
|
|
18
|
+
yield session
|
|
File without changes
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
from sqlalchemy import (
|
|
2
|
+
Column,
|
|
3
|
+
DateTime,
|
|
4
|
+
String,
|
|
5
|
+
bindparam,
|
|
6
|
+
delete,
|
|
7
|
+
func,
|
|
8
|
+
insert,
|
|
9
|
+
or_,
|
|
10
|
+
select,
|
|
11
|
+
update,
|
|
12
|
+
)
|
|
13
|
+
from sqlalchemy.orm import relationship
|
|
14
|
+
|
|
15
|
+
from ..db import db_sessionmaker
|
|
16
|
+
from ..schemas.db.celery_tasks import CeleryTasksDbSchema
|
|
17
|
+
from .base import CeleryTasksScheduleBase
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class CeleryTasksModel(CeleryTasksScheduleBase):
|
|
21
|
+
"""Application celery tasks."""
|
|
22
|
+
|
|
23
|
+
__tablename__ = "celery_tasks"
|
|
24
|
+
|
|
25
|
+
task = Column(String, primary_key=True, unique=True)
|
|
26
|
+
params = Column(String, nullable=False, server_default="")
|
|
27
|
+
description = Column(String, nullable=False, server_default="")
|
|
28
|
+
tags = Column(String, nullable=False, server_default="")
|
|
29
|
+
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
|
30
|
+
updated_at = Column(DateTime(timezone=True), onupdate=func.now())
|
|
31
|
+
|
|
32
|
+
scheduled_tasks = relationship(
|
|
33
|
+
"CeleryTasksScheduleModel",
|
|
34
|
+
back_populates="task_head",
|
|
35
|
+
lazy="selectin",
|
|
36
|
+
cascade="all, delete-orphan",
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def bulk_insert(cls, data: list[CeleryTasksDbSchema]):
|
|
41
|
+
with db_sessionmaker() as session, session.begin():
|
|
42
|
+
existing_tasks = set((session.execute(select(cls.task))).scalars().all())
|
|
43
|
+
tasks_to_insert = {d.task for d in data}
|
|
44
|
+
missing_tasks = tasks_to_insert - existing_tasks
|
|
45
|
+
if missing_tasks:
|
|
46
|
+
stmt = insert(cls).values(**CeleryTasksDbSchema.get_bindparams())
|
|
47
|
+
session.execute(stmt, [d.model_dump() for d in data if d.task in missing_tasks])
|
|
48
|
+
|
|
49
|
+
@classmethod
|
|
50
|
+
def bulk_update(cls, data: list[CeleryTasksDbSchema]):
|
|
51
|
+
with db_sessionmaker() as session, session.begin():
|
|
52
|
+
stmt = (
|
|
53
|
+
update(cls)
|
|
54
|
+
.where(
|
|
55
|
+
cls.task == bindparam("task_value"),
|
|
56
|
+
or_(
|
|
57
|
+
cls.params != bindparam("params_value"),
|
|
58
|
+
cls.description != bindparam("description_value"),
|
|
59
|
+
),
|
|
60
|
+
)
|
|
61
|
+
.values(**CeleryTasksDbSchema.get_bindparams(exclude=["tags"]))
|
|
62
|
+
.execution_options(synchronize_session=None)
|
|
63
|
+
)
|
|
64
|
+
session.connection().execute(stmt, [d.model_dump() for d in data])
|
|
65
|
+
|
|
66
|
+
@classmethod
|
|
67
|
+
def select_all_tasks(cls) -> list[str]:
|
|
68
|
+
with db_sessionmaker() as session, session.begin():
|
|
69
|
+
stmt = select(cls.task)
|
|
70
|
+
return (session.execute(stmt)).scalars().all()
|
|
71
|
+
|
|
72
|
+
@classmethod
|
|
73
|
+
def delete(cls, tasks: list[str]):
|
|
74
|
+
with db_sessionmaker() as session, session.begin():
|
|
75
|
+
stmt = delete(cls).where(cls.task.in_(tasks))
|
|
76
|
+
session.execute(stmt)
|
|
77
|
+
|
|
78
|
+
def __repr__(self):
|
|
79
|
+
return f"{self.task}"
|
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import json
|
|
2
|
+
from json import JSONDecodeError
|
|
3
|
+
|
|
4
|
+
from celery import current_app as celery_app
|
|
5
|
+
from celery.beat import debug
|
|
6
|
+
from celery.schedules import crontab
|
|
7
|
+
from sqlalchemy import (
|
|
8
|
+
BigInteger,
|
|
9
|
+
Boolean,
|
|
10
|
+
Column,
|
|
11
|
+
DateTime,
|
|
12
|
+
ForeignKey,
|
|
13
|
+
String,
|
|
14
|
+
delete,
|
|
15
|
+
event,
|
|
16
|
+
func,
|
|
17
|
+
insert,
|
|
18
|
+
select,
|
|
19
|
+
true,
|
|
20
|
+
)
|
|
21
|
+
from sqlalchemy.orm import relationship
|
|
22
|
+
|
|
23
|
+
from ..db import db_sessionmaker
|
|
24
|
+
from ..schemas.db.celery_tasks_schedule import CeleryTasksScheduleDbSchema
|
|
25
|
+
from ..utils import get_task_key
|
|
26
|
+
from .base import CeleryTasksScheduleBase
|
|
27
|
+
from .celery_tasks_schedule_meta import CeleryTasksScheduleMetaModel
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class CeleryTasksScheduleModel(CeleryTasksScheduleBase):
|
|
31
|
+
"""Celery tasks schedule."""
|
|
32
|
+
|
|
33
|
+
__tablename__ = "celery_tasks_schedule"
|
|
34
|
+
|
|
35
|
+
id = Column(BigInteger, primary_key=True)
|
|
36
|
+
task = Column(
|
|
37
|
+
String,
|
|
38
|
+
ForeignKey("celery_tasks.task", ondelete="CASCADE"),
|
|
39
|
+
nullable=False,
|
|
40
|
+
doc="Task name (Example: my_task)",
|
|
41
|
+
)
|
|
42
|
+
args = Column(
|
|
43
|
+
String,
|
|
44
|
+
nullable=False,
|
|
45
|
+
server_default="[]",
|
|
46
|
+
default="[]",
|
|
47
|
+
doc='Task positional arguments (Example: ["arg1", 2])',
|
|
48
|
+
)
|
|
49
|
+
kwargs = Column(
|
|
50
|
+
String,
|
|
51
|
+
nullable=False,
|
|
52
|
+
server_default="{}",
|
|
53
|
+
default="{}",
|
|
54
|
+
doc='Task keyword arguments (Example: {"kwarg1": true})',
|
|
55
|
+
)
|
|
56
|
+
schedule = Column(
|
|
57
|
+
String,
|
|
58
|
+
nullable=False,
|
|
59
|
+
doc="Crontab schedule [minute hour day_of_month month day_of_week] (Example: */2 * * * *)",
|
|
60
|
+
)
|
|
61
|
+
enabled = Column(
|
|
62
|
+
Boolean,
|
|
63
|
+
nullable=False,
|
|
64
|
+
server_default=true(),
|
|
65
|
+
default="checked",
|
|
66
|
+
doc="Enable/disable task execution",
|
|
67
|
+
)
|
|
68
|
+
created_at = Column(
|
|
69
|
+
DateTime(timezone=True), server_default=func.now(), doc="Datetime this task was created"
|
|
70
|
+
)
|
|
71
|
+
updated_at = Column(
|
|
72
|
+
DateTime(timezone=True), onupdate=func.now(), doc="Datetime this task was last modified"
|
|
73
|
+
)
|
|
74
|
+
comment = Column(String, nullable=True, doc="Any comment")
|
|
75
|
+
task_key = Column(String, nullable=False, unique=True)
|
|
76
|
+
|
|
77
|
+
task_head = relationship("CeleryTasksModel", back_populates="scheduled_tasks", lazy="selectin")
|
|
78
|
+
|
|
79
|
+
@classmethod
|
|
80
|
+
def bulk_insert(cls, data: list[CeleryTasksScheduleDbSchema]):
|
|
81
|
+
with db_sessionmaker() as session, session.begin():
|
|
82
|
+
existing_entries = set((session.execute(select(cls.task_key))).scalars().all())
|
|
83
|
+
entries_to_insert = {d.task_key for d in data}
|
|
84
|
+
missing_entries = entries_to_insert - existing_entries
|
|
85
|
+
if missing_entries:
|
|
86
|
+
stmt = insert(cls).values(**CeleryTasksScheduleDbSchema.get_bindparams())
|
|
87
|
+
session.execute(
|
|
88
|
+
stmt, [d.model_dump() for d in data if d.task_key in missing_entries]
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
@classmethod
|
|
92
|
+
def select_all_tasks(cls) -> list[str]:
|
|
93
|
+
with db_sessionmaker() as session, session.begin():
|
|
94
|
+
stmt = select(cls.task)
|
|
95
|
+
return (session.execute(stmt)).scalars().all()
|
|
96
|
+
|
|
97
|
+
@classmethod
|
|
98
|
+
def select_enabled_entries(cls):
|
|
99
|
+
with db_sessionmaker() as session, session.begin():
|
|
100
|
+
stmt = select(cls).where(cls.enabled)
|
|
101
|
+
return (session.execute(stmt)).scalars().all()
|
|
102
|
+
|
|
103
|
+
@classmethod
|
|
104
|
+
def delete(cls, tasks: list[str]):
|
|
105
|
+
with db_sessionmaker() as session, session.begin():
|
|
106
|
+
stmt = delete(cls).where(cls.task.in_(tasks))
|
|
107
|
+
session.execute(stmt)
|
|
108
|
+
|
|
109
|
+
def __repr__(self):
|
|
110
|
+
return (
|
|
111
|
+
f"Task: {self.task} Args: {self.args} Kwargs: {self.kwargs} Schedule: {self.schedule}"
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@event.listens_for(CeleryTasksScheduleModel, "before_insert")
|
|
116
|
+
def before_insert_handler(mapper, connection, target: CeleryTasksScheduleModel):
|
|
117
|
+
before_validator(mapper, connection, target)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@event.listens_for(CeleryTasksScheduleModel, "before_update")
|
|
121
|
+
def before_update_handler(mapper, connection, target: CeleryTasksScheduleModel):
|
|
122
|
+
before_validator(mapper, connection, target)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
@event.listens_for(CeleryTasksScheduleModel, "after_insert")
|
|
126
|
+
def after_insert_handler(mapper, connection, target: CeleryTasksScheduleModel):
|
|
127
|
+
CeleryTasksScheduleMetaModel.update_last_updated_at()
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@event.listens_for(CeleryTasksScheduleModel, "after_update")
|
|
131
|
+
def after_update_handler(mapper, connection, target: CeleryTasksScheduleModel):
|
|
132
|
+
CeleryTasksScheduleMetaModel.update_last_updated_at()
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@event.listens_for(CeleryTasksScheduleModel, "after_delete")
|
|
136
|
+
def after_delete_handler(mapper, connection, target: CeleryTasksScheduleModel):
|
|
137
|
+
CeleryTasksScheduleMetaModel.update_last_updated_at()
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def before_validator(mapper, connection, target):
|
|
141
|
+
if target.task not in celery_app.tasks:
|
|
142
|
+
raise RuntimeError(f"Task '{target.task}' does not exist")
|
|
143
|
+
|
|
144
|
+
try:
|
|
145
|
+
args = json.loads(target.args)
|
|
146
|
+
except JSONDecodeError as e:
|
|
147
|
+
raise RuntimeError(f"Invalid format for task positional arguments: {e}")
|
|
148
|
+
|
|
149
|
+
try:
|
|
150
|
+
kwargs = json.loads(target.kwargs)
|
|
151
|
+
except JSONDecodeError as e:
|
|
152
|
+
raise RuntimeError(f"Invalid format for task keyword arguments: {e}")
|
|
153
|
+
|
|
154
|
+
task = celery_app.tasks[target.task]
|
|
155
|
+
try:
|
|
156
|
+
check_arguments = task.__header__
|
|
157
|
+
except AttributeError: # pragma: no cover
|
|
158
|
+
debug(f"Task '{task}' has no attribute '__header__': cannot validate arguments")
|
|
159
|
+
else:
|
|
160
|
+
check_arguments(*(args or ()), **(kwargs or {}))
|
|
161
|
+
|
|
162
|
+
try:
|
|
163
|
+
minute, hour, day_of_month, month_of_year, day_of_week = target.schedule.split()
|
|
164
|
+
crontab(
|
|
165
|
+
minute=minute,
|
|
166
|
+
hour=hour,
|
|
167
|
+
day_of_week=day_of_week,
|
|
168
|
+
day_of_month=day_of_month,
|
|
169
|
+
month_of_year=month_of_year,
|
|
170
|
+
)
|
|
171
|
+
except Exception as e:
|
|
172
|
+
raise RuntimeError(f"Invalid schedule: {e}")
|
|
173
|
+
|
|
174
|
+
# Generate unique task key
|
|
175
|
+
target.task_key = get_task_key(target.task, args, kwargs)
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
import datetime as dt
|
|
2
|
+
|
|
3
|
+
from sqlalchemy import (
|
|
4
|
+
BigInteger,
|
|
5
|
+
Column,
|
|
6
|
+
DateTime,
|
|
7
|
+
func,
|
|
8
|
+
insert,
|
|
9
|
+
select,
|
|
10
|
+
update,
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
from ..db import db_sessionmaker
|
|
14
|
+
from .base import CeleryTasksScheduleBase
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class CeleryTasksScheduleMetaModel(CeleryTasksScheduleBase):
|
|
18
|
+
"""Keeps time when celery_tasks_schedule table was last updated."""
|
|
19
|
+
|
|
20
|
+
__tablename__ = "celery_tasks_schedule_meta"
|
|
21
|
+
|
|
22
|
+
id = Column(BigInteger, primary_key=True)
|
|
23
|
+
last_updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
|
|
24
|
+
|
|
25
|
+
@classmethod
|
|
26
|
+
def init_last_updated_at(cls):
|
|
27
|
+
with db_sessionmaker() as session, session.begin():
|
|
28
|
+
entry = (session.execute(select(cls).where(cls.id == 1))).scalars().first()
|
|
29
|
+
if not entry:
|
|
30
|
+
stmt = insert(cls).values(id=1, last_updated_at=dt.datetime.now(tz=dt.UTC))
|
|
31
|
+
session.execute(stmt)
|
|
32
|
+
|
|
33
|
+
@classmethod
|
|
34
|
+
def update_last_updated_at(cls):
|
|
35
|
+
with db_sessionmaker() as session, session.begin():
|
|
36
|
+
stmt = update(cls).where(cls.id == 1).values(last_updated_at=dt.datetime.now(tz=dt.UTC))
|
|
37
|
+
session.execute(stmt)
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def get_last_updated_at(cls) -> dt.datetime:
|
|
41
|
+
with db_sessionmaker() as session, session.begin():
|
|
42
|
+
stmt = select(cls.last_updated_at).where(cls.id == 1)
|
|
43
|
+
return (session.execute(stmt)).scalars().first()
|
|
44
|
+
|
|
45
|
+
def __repr__(self):
|
|
46
|
+
return f"{self.last_updated_at}"
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
import datetime as dt
|
|
2
|
+
import enum
|
|
3
|
+
import inspect
|
|
4
|
+
import json
|
|
5
|
+
import time
|
|
6
|
+
|
|
7
|
+
from celery.beat import ScheduleEntry, Scheduler, debug, error
|
|
8
|
+
from celery.schedules import crontab
|
|
9
|
+
from celery.utils.time import get_exponential_backoff_interval
|
|
10
|
+
from sqlalchemy.exc import DatabaseError
|
|
11
|
+
|
|
12
|
+
from .db import engine
|
|
13
|
+
from .models.base import CeleryTasksScheduleBase
|
|
14
|
+
from .models.celery_tasks import CeleryTasksModel
|
|
15
|
+
from .models.celery_tasks_schedule import CeleryTasksScheduleModel
|
|
16
|
+
from .models.celery_tasks_schedule_meta import CeleryTasksScheduleMetaModel
|
|
17
|
+
from .schemas.db.celery_tasks import CeleryTasksDbSchema
|
|
18
|
+
from .schemas.db.celery_tasks_schedule import CeleryTasksScheduleDbSchema
|
|
19
|
+
from .utils import get_task_category, get_task_key
|
|
20
|
+
|
|
21
|
+
DEFAULT_MAX_LOOP_INTERVAL = 5 # seconds
|
|
22
|
+
MAX_SYNC_SCHEDULE_INTERVAL = 5 # minutes
|
|
23
|
+
PREPARE_MODELS_MAX_RETRIES = 10
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DatabaseScheduler(Scheduler):
|
|
27
|
+
"""Database-backed Beat Scheduler."""
|
|
28
|
+
|
|
29
|
+
ModelTask = CeleryTasksModel
|
|
30
|
+
ModelSchedule = CeleryTasksScheduleModel
|
|
31
|
+
ModelMeta = CeleryTasksScheduleMetaModel
|
|
32
|
+
|
|
33
|
+
_schedule = None
|
|
34
|
+
_last_updated_at = None
|
|
35
|
+
|
|
36
|
+
def __init__(self, *args, **kwargs):
|
|
37
|
+
"""Initialize the database scheduler."""
|
|
38
|
+
Scheduler.__init__(self, *args, **kwargs)
|
|
39
|
+
self.max_interval = (
|
|
40
|
+
kwargs.get("max_interval")
|
|
41
|
+
or self.app.conf.beat_max_loop_interval
|
|
42
|
+
or DEFAULT_MAX_LOOP_INTERVAL
|
|
43
|
+
)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def schedule(self):
|
|
47
|
+
self._schedule = self._schedule or {}
|
|
48
|
+
if self._need_update():
|
|
49
|
+
self._update_schedule()
|
|
50
|
+
return self._schedule
|
|
51
|
+
|
|
52
|
+
def setup_schedule(self):
|
|
53
|
+
debug("Setup schedule")
|
|
54
|
+
self._prepare_models()
|
|
55
|
+
self._clean_deprecated()
|
|
56
|
+
self._fill_celery_tasks()
|
|
57
|
+
self._fill_celery_tasks_schedule()
|
|
58
|
+
self._update_schedule()
|
|
59
|
+
|
|
60
|
+
def _fill_celery_tasks(self):
|
|
61
|
+
"""Fill celery_tasks table with application celery tasks."""
|
|
62
|
+
|
|
63
|
+
def convert_type(param_type) -> str:
|
|
64
|
+
if param_type is inspect._empty:
|
|
65
|
+
return ""
|
|
66
|
+
if type(param_type) is type:
|
|
67
|
+
return param_type.__name__
|
|
68
|
+
if type(param_type) is enum.EnumType:
|
|
69
|
+
return f"<one of: {", ".join(set(st.value for st in param_type))}>"
|
|
70
|
+
return str(param_type)
|
|
71
|
+
|
|
72
|
+
def get_annotation(param_annotation) -> str:
|
|
73
|
+
if param_annotation is inspect._empty:
|
|
74
|
+
return ""
|
|
75
|
+
return f": {convert_type(param_annotation)}"
|
|
76
|
+
|
|
77
|
+
def get_default(param_default) -> str:
|
|
78
|
+
if param_default is inspect._empty:
|
|
79
|
+
return ""
|
|
80
|
+
return f" [default: {convert_type(param_default)}]"
|
|
81
|
+
|
|
82
|
+
if self.app.tasks:
|
|
83
|
+
db_entries = [
|
|
84
|
+
CeleryTasksDbSchema(
|
|
85
|
+
task=task_name,
|
|
86
|
+
params=", ".join(
|
|
87
|
+
[
|
|
88
|
+
f"{name}{get_annotation(param.annotation)}{get_default(param.default)}"
|
|
89
|
+
for name, param in inspect.signature(task).parameters.items()
|
|
90
|
+
if name != "kwargs"
|
|
91
|
+
]
|
|
92
|
+
),
|
|
93
|
+
description=inspect.getdoc(task) if task.__doc__ else "",
|
|
94
|
+
tags=get_task_category(task.__module__),
|
|
95
|
+
)
|
|
96
|
+
for task_name, task in sorted(self.app.tasks.items(), key=lambda item: item[0])
|
|
97
|
+
if not task_name.startswith("celery.")
|
|
98
|
+
]
|
|
99
|
+
self.ModelTask.bulk_insert(data=db_entries)
|
|
100
|
+
self.ModelTask.bulk_update(data=db_entries)
|
|
101
|
+
|
|
102
|
+
def _fill_celery_tasks_schedule(self):
|
|
103
|
+
"""Fill celery_tasks_schedule table with application beat schedule."""
|
|
104
|
+
if self.app.conf.beat_schedule:
|
|
105
|
+
comment = (
|
|
106
|
+
f"loaded from app celery beat schedule on {dt.datetime.now(tz=dt.UTC).isoformat()}"
|
|
107
|
+
)
|
|
108
|
+
db_entries = [
|
|
109
|
+
CeleryTasksScheduleDbSchema(
|
|
110
|
+
task_key=get_task_key(task["task"], task["args"], task["kwargs"]),
|
|
111
|
+
task=task["task"],
|
|
112
|
+
args=json.dumps(task["args"]),
|
|
113
|
+
kwargs=json.dumps(task["kwargs"]),
|
|
114
|
+
schedule=(
|
|
115
|
+
f"{task["schedule"]._orig_minute} "
|
|
116
|
+
f"{task["schedule"]._orig_hour} "
|
|
117
|
+
f"{task["schedule"]._orig_day_of_month} "
|
|
118
|
+
f"{task["schedule"]._orig_month_of_year} "
|
|
119
|
+
f"{task["schedule"]._orig_day_of_week}"
|
|
120
|
+
),
|
|
121
|
+
comment=comment,
|
|
122
|
+
)
|
|
123
|
+
for task in self.app.conf.beat_schedule.values()
|
|
124
|
+
]
|
|
125
|
+
self.ModelSchedule.bulk_insert(data=db_entries)
|
|
126
|
+
|
|
127
|
+
def update_from_dict(self, dict_):
|
|
128
|
+
self._schedule = self._schedule or {}
|
|
129
|
+
self._schedule.update(
|
|
130
|
+
{name: self._maybe_entry(name, entry) for name, entry in dict_.items()}
|
|
131
|
+
)
|
|
132
|
+
|
|
133
|
+
def _need_update(self) -> bool:
|
|
134
|
+
now = dt.datetime.now(tz=dt.UTC)
|
|
135
|
+
|
|
136
|
+
if self.max_interval < 30: # noqa
|
|
137
|
+
# Don't update at the beginning and at the end of a minute
|
|
138
|
+
# to avoid overriding current heap
|
|
139
|
+
if 0 <= now.second < 20 or now.second > 50: # noqa
|
|
140
|
+
return False
|
|
141
|
+
|
|
142
|
+
# Sync beat schedule with db every 5 minutes
|
|
143
|
+
# in case we didn't get sqlalchemy update event
|
|
144
|
+
sync_threshold = now - dt.timedelta(minutes=MAX_SYNC_SCHEDULE_INTERVAL)
|
|
145
|
+
|
|
146
|
+
if ( # noqa
|
|
147
|
+
self._last_updated_at is None
|
|
148
|
+
or self._last_updated_at < sync_threshold
|
|
149
|
+
or self._last_updated_at < self.ModelMeta.get_last_updated_at()
|
|
150
|
+
):
|
|
151
|
+
return True
|
|
152
|
+
return False
|
|
153
|
+
|
|
154
|
+
def _update_schedule(self):
|
|
155
|
+
enabled_tasks = self._get_enabled_tasks()
|
|
156
|
+
self._schedule = enabled_tasks
|
|
157
|
+
self._last_updated_at = dt.datetime.now(tz=dt.UTC)
|
|
158
|
+
self.install_default_entries(self._schedule)
|
|
159
|
+
debug("Current schedule:\n" + "\n".join(repr(entry) for entry in self._schedule.values()))
|
|
160
|
+
|
|
161
|
+
def _get_enabled_tasks(self) -> dict[str, ScheduleEntry]:
|
|
162
|
+
"""Return list of enabled periodic tasks."""
|
|
163
|
+
rows = self.ModelSchedule.select_enabled_entries()
|
|
164
|
+
entries = {}
|
|
165
|
+
for row in rows:
|
|
166
|
+
try:
|
|
167
|
+
entries[row.task_key] = self._model_to_entry(model=row)
|
|
168
|
+
except Exception as e:
|
|
169
|
+
error(str(e))
|
|
170
|
+
return entries
|
|
171
|
+
|
|
172
|
+
def _model_to_entry(self, model: CeleryTasksScheduleModel) -> ScheduleEntry:
|
|
173
|
+
args = json.loads(model.args)
|
|
174
|
+
kwargs = json.loads(model.kwargs)
|
|
175
|
+
minute, hour, day_of_month, month_of_year, day_of_week = model.schedule.split()
|
|
176
|
+
schedule = crontab(
|
|
177
|
+
minute=minute,
|
|
178
|
+
hour=hour,
|
|
179
|
+
day_of_week=day_of_week,
|
|
180
|
+
day_of_month=day_of_month,
|
|
181
|
+
month_of_year=month_of_year,
|
|
182
|
+
)
|
|
183
|
+
entry_dict = {
|
|
184
|
+
"task": model.task,
|
|
185
|
+
"args": args,
|
|
186
|
+
"kwargs": kwargs,
|
|
187
|
+
"schedule": schedule,
|
|
188
|
+
}
|
|
189
|
+
return self._maybe_entry(model.task_key, entry_dict)
|
|
190
|
+
|
|
191
|
+
def _prepare_models(self):
|
|
192
|
+
# ###
|
|
193
|
+
# COPIED from: celery.backends.database.session.SessionManager.prepare_models
|
|
194
|
+
# ###
|
|
195
|
+
# SQLAlchemy will check if the items exist before trying to
|
|
196
|
+
# create them, which is a race condition. If it raises an error
|
|
197
|
+
# in one iteration, the next may pass all the existence checks
|
|
198
|
+
# and the call will succeed.
|
|
199
|
+
retries = 0
|
|
200
|
+
while True:
|
|
201
|
+
try:
|
|
202
|
+
CeleryTasksScheduleBase.metadata.create_all(engine)
|
|
203
|
+
except DatabaseError:
|
|
204
|
+
if retries < PREPARE_MODELS_MAX_RETRIES:
|
|
205
|
+
sleep_amount_ms = get_exponential_backoff_interval(10, retries, 1000, True)
|
|
206
|
+
time.sleep(sleep_amount_ms / 1000)
|
|
207
|
+
retries += 1
|
|
208
|
+
else:
|
|
209
|
+
raise
|
|
210
|
+
else:
|
|
211
|
+
break
|
|
212
|
+
|
|
213
|
+
self.ModelMeta.init_last_updated_at()
|
|
214
|
+
|
|
215
|
+
def _clean_deprecated(self):
|
|
216
|
+
"""Sync celery_tasks table with application tasks."""
|
|
217
|
+
tasks = set(self.app.tasks)
|
|
218
|
+
db_tasks = set(self.ModelTask.select_all_tasks())
|
|
219
|
+
deprecated_tasks = db_tasks - tasks
|
|
220
|
+
if deprecated_tasks:
|
|
221
|
+
self.ModelTask.delete(tasks=list(deprecated_tasks))
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
from dataclasses import dataclass, fields
|
|
2
|
+
|
|
3
|
+
from sqlalchemy import bindparam
|
|
4
|
+
|
|
5
|
+
from ...utils import db_bindparam
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
@dataclass
|
|
9
|
+
class BindparamDbSchema:
|
|
10
|
+
@classmethod
|
|
11
|
+
def get_bindparams(cls, exclude: list[str] | None = None):
|
|
12
|
+
exclude = exclude or []
|
|
13
|
+
return {
|
|
14
|
+
f.name: bindparam(db_bindparam(f.name)) for f in fields(cls) if f.name not in exclude
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
def model_dump(self):
|
|
18
|
+
return {db_bindparam(f.name): getattr(self, f.name) for f in fields(self)}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
from dataclasses import dataclass
|
|
2
|
+
|
|
3
|
+
from .base import BindparamDbSchema
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
@dataclass
|
|
7
|
+
class CeleryTasksScheduleDbSchema(BindparamDbSchema):
|
|
8
|
+
task_key: str
|
|
9
|
+
task: str
|
|
10
|
+
args: str
|
|
11
|
+
kwargs: str
|
|
12
|
+
schedule: str
|
|
13
|
+
enabled: bool = True
|
|
14
|
+
comment: str | None = None
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import re
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
def get_task_key(name, args, kwargs) -> str:
|
|
5
|
+
"""
|
|
6
|
+
Get task unique identifier.
|
|
7
|
+
The same task can be run with different args/kwargs, and they are different entries for celery beat.
|
|
8
|
+
To uniquely identify those entries we use combination of task_name-args-kwargs.
|
|
9
|
+
"""
|
|
10
|
+
args = "-".join(map(str, args))
|
|
11
|
+
kwargs = "-".join([f"{k}-{v}" for k, v in kwargs.items()])
|
|
12
|
+
# Zabbix compatible key format: 0-9a-zA-Z_-.
|
|
13
|
+
return re.sub(r"[\[\](){}, '\"]", "", f"{name}-{args}-{kwargs}".strip("-"))
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def get_task_category(task_module: str):
|
|
17
|
+
"""
|
|
18
|
+
Get task category by its module:
|
|
19
|
+
src.apps.users.tasks -> users;
|
|
20
|
+
src.apps.base.tasks.tasks -> base.
|
|
21
|
+
"""
|
|
22
|
+
return task_module.replace(".tasks", "").split(".")[-1]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def db_bindparam(name: str) -> str:
|
|
26
|
+
return f"{name}_value"
|