celery-beat-sqlalchemy 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,8 @@
1
+ venv
2
+ .venv
3
+
4
+ # Python stuff
5
+ __pycache__
6
+
7
+ # Pycharm
8
+ .idea
@@ -0,0 +1,19 @@
1
+ Copyright (c) 2026 sqlalchemy_celery_beat
2
+
3
+ Permission is hereby granted, free of charge, to any person obtaining a copy
4
+ of this software and associated documentation files (the «Software»), to deal
5
+ in the Software without restriction, including without limitation the rights
6
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7
+ copies of the Software, and to permit persons to whom the Software is
8
+ furnished to do so, subject to the following conditions:
9
+
10
+ The above copyright notice and this permission notice shall be included in all
11
+ copies or substantial portions of the Software.
12
+
13
+ THE SOFTWARE IS PROVIDED «AS IS», WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19
+ SOFTWARE.
@@ -0,0 +1,13 @@
1
+ Metadata-Version: 2.4
2
+ Name: celery-beat-sqlalchemy
3
+ Version: 0.0.1
4
+ Summary: Library for managing celery beat schedule through sqlalchemy models
5
+ Project-URL: Homepage, https://github.com/ebergard/sqlalchemy_celery_beat
6
+ Author-email: Kseniia Ebergard <ebergard-xu@bk.ru>
7
+ License-File: LICENSE
8
+ Classifier: License :: OSI Approved :: MIT License
9
+ Classifier: Operating System :: OS Independent
10
+ Classifier: Programming Language :: Python :: 3
11
+ Requires-Python: >=3.8.0
12
+ Requires-Dist: celery>=5.4.0
13
+ Requires-Dist: sqlalchemy>=2.0.0
File without changes
@@ -0,0 +1,25 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "celery-beat-sqlalchemy"
7
+ version = "0.0.1"
8
+ authors = [
9
+ { name="Kseniia Ebergard", email="ebergard-xu@bk.ru" },
10
+ ]
11
+ description = "Library for managing celery beat schedule through sqlalchemy models"
12
+ readme = "README.md"
13
+ requires-python = ">=3.8.0"
14
+ dependencies = [
15
+ "celery>=5.4.0",
16
+ "sqlalchemy>=2.0.0"
17
+ ]
18
+ classifiers = [
19
+ "Programming Language :: Python :: 3",
20
+ "License :: OSI Approved :: MIT License",
21
+ "Operating System :: OS Independent",
22
+ ]
23
+
24
+ [project.urls]
25
+ Homepage = "https://github.com/ebergard/sqlalchemy_celery_beat"
@@ -0,0 +1,18 @@
1
+ import contextlib
2
+ from collections.abc import Generator
3
+ from typing import Any
4
+
5
+ from celery import current_app as celery_app
6
+ from sqlalchemy import NullPool, create_engine
7
+ from sqlalchemy.orm import Session
8
+
9
+ engine = create_engine(url=celery_app.conf.beat_db_scheduler_dsn, future=True, poolclass=NullPool)
10
+
11
+
12
+ @contextlib.contextmanager
13
+ def db_sessionmaker() -> Generator[Session, Any, None]:
14
+ with (
15
+ engine.connect() as conn,
16
+ Session(bind=conn, expire_on_commit=False, autoflush=False) as session,
17
+ ):
18
+ yield session
@@ -0,0 +1,3 @@
1
+ from sqlalchemy.orm import declarative_base
2
+
3
+ CeleryTasksScheduleBase = declarative_base()
@@ -0,0 +1,79 @@
1
+ from sqlalchemy import (
2
+ Column,
3
+ DateTime,
4
+ String,
5
+ bindparam,
6
+ delete,
7
+ func,
8
+ insert,
9
+ or_,
10
+ select,
11
+ update,
12
+ )
13
+ from sqlalchemy.orm import relationship
14
+
15
+ from ..db import db_sessionmaker
16
+ from ..schemas.db.celery_tasks import CeleryTasksDbSchema
17
+ from .base import CeleryTasksScheduleBase
18
+
19
+
20
+ class CeleryTasksModel(CeleryTasksScheduleBase):
21
+ """Application celery tasks."""
22
+
23
+ __tablename__ = "celery_tasks"
24
+
25
+ task = Column(String, primary_key=True, unique=True)
26
+ params = Column(String, nullable=False, server_default="")
27
+ description = Column(String, nullable=False, server_default="")
28
+ tags = Column(String, nullable=False, server_default="")
29
+ created_at = Column(DateTime(timezone=True), server_default=func.now())
30
+ updated_at = Column(DateTime(timezone=True), onupdate=func.now())
31
+
32
+ scheduled_tasks = relationship(
33
+ "CeleryTasksScheduleModel",
34
+ back_populates="task_head",
35
+ lazy="selectin",
36
+ cascade="all, delete-orphan",
37
+ )
38
+
39
+ @classmethod
40
+ def bulk_insert(cls, data: list[CeleryTasksDbSchema]):
41
+ with db_sessionmaker() as session, session.begin():
42
+ existing_tasks = set((session.execute(select(cls.task))).scalars().all())
43
+ tasks_to_insert = {d.task for d in data}
44
+ missing_tasks = tasks_to_insert - existing_tasks
45
+ if missing_tasks:
46
+ stmt = insert(cls).values(**CeleryTasksDbSchema.get_bindparams())
47
+ session.execute(stmt, [d.model_dump() for d in data if d.task in missing_tasks])
48
+
49
+ @classmethod
50
+ def bulk_update(cls, data: list[CeleryTasksDbSchema]):
51
+ with db_sessionmaker() as session, session.begin():
52
+ stmt = (
53
+ update(cls)
54
+ .where(
55
+ cls.task == bindparam("task_value"),
56
+ or_(
57
+ cls.params != bindparam("params_value"),
58
+ cls.description != bindparam("description_value"),
59
+ ),
60
+ )
61
+ .values(**CeleryTasksDbSchema.get_bindparams(exclude=["tags"]))
62
+ .execution_options(synchronize_session=None)
63
+ )
64
+ session.connection().execute(stmt, [d.model_dump() for d in data])
65
+
66
+ @classmethod
67
+ def select_all_tasks(cls) -> list[str]:
68
+ with db_sessionmaker() as session, session.begin():
69
+ stmt = select(cls.task)
70
+ return (session.execute(stmt)).scalars().all()
71
+
72
+ @classmethod
73
+ def delete(cls, tasks: list[str]):
74
+ with db_sessionmaker() as session, session.begin():
75
+ stmt = delete(cls).where(cls.task.in_(tasks))
76
+ session.execute(stmt)
77
+
78
+ def __repr__(self):
79
+ return f"{self.task}"
@@ -0,0 +1,175 @@
1
+ import json
2
+ from json import JSONDecodeError
3
+
4
+ from celery import current_app as celery_app
5
+ from celery.beat import debug
6
+ from celery.schedules import crontab
7
+ from sqlalchemy import (
8
+ BigInteger,
9
+ Boolean,
10
+ Column,
11
+ DateTime,
12
+ ForeignKey,
13
+ String,
14
+ delete,
15
+ event,
16
+ func,
17
+ insert,
18
+ select,
19
+ true,
20
+ )
21
+ from sqlalchemy.orm import relationship
22
+
23
+ from ..db import db_sessionmaker
24
+ from ..schemas.db.celery_tasks_schedule import CeleryTasksScheduleDbSchema
25
+ from ..utils import get_task_key
26
+ from .base import CeleryTasksScheduleBase
27
+ from .celery_tasks_schedule_meta import CeleryTasksScheduleMetaModel
28
+
29
+
30
+ class CeleryTasksScheduleModel(CeleryTasksScheduleBase):
31
+ """Celery tasks schedule."""
32
+
33
+ __tablename__ = "celery_tasks_schedule"
34
+
35
+ id = Column(BigInteger, primary_key=True)
36
+ task = Column(
37
+ String,
38
+ ForeignKey("celery_tasks.task", ondelete="CASCADE"),
39
+ nullable=False,
40
+ doc="Task name (Example: my_task)",
41
+ )
42
+ args = Column(
43
+ String,
44
+ nullable=False,
45
+ server_default="[]",
46
+ default="[]",
47
+ doc='Task positional arguments (Example: ["arg1", 2])',
48
+ )
49
+ kwargs = Column(
50
+ String,
51
+ nullable=False,
52
+ server_default="{}",
53
+ default="{}",
54
+ doc='Task keyword arguments (Example: {"kwarg1": true})',
55
+ )
56
+ schedule = Column(
57
+ String,
58
+ nullable=False,
59
+ doc="Crontab schedule [minute hour day_of_month month day_of_week] (Example: */2 * * * *)",
60
+ )
61
+ enabled = Column(
62
+ Boolean,
63
+ nullable=False,
64
+ server_default=true(),
65
+ default="checked",
66
+ doc="Enable/disable task execution",
67
+ )
68
+ created_at = Column(
69
+ DateTime(timezone=True), server_default=func.now(), doc="Datetime this task was created"
70
+ )
71
+ updated_at = Column(
72
+ DateTime(timezone=True), onupdate=func.now(), doc="Datetime this task was last modified"
73
+ )
74
+ comment = Column(String, nullable=True, doc="Any comment")
75
+ task_key = Column(String, nullable=False, unique=True)
76
+
77
+ task_head = relationship("CeleryTasksModel", back_populates="scheduled_tasks", lazy="selectin")
78
+
79
+ @classmethod
80
+ def bulk_insert(cls, data: list[CeleryTasksScheduleDbSchema]):
81
+ with db_sessionmaker() as session, session.begin():
82
+ existing_entries = set((session.execute(select(cls.task_key))).scalars().all())
83
+ entries_to_insert = {d.task_key for d in data}
84
+ missing_entries = entries_to_insert - existing_entries
85
+ if missing_entries:
86
+ stmt = insert(cls).values(**CeleryTasksScheduleDbSchema.get_bindparams())
87
+ session.execute(
88
+ stmt, [d.model_dump() for d in data if d.task_key in missing_entries]
89
+ )
90
+
91
+ @classmethod
92
+ def select_all_tasks(cls) -> list[str]:
93
+ with db_sessionmaker() as session, session.begin():
94
+ stmt = select(cls.task)
95
+ return (session.execute(stmt)).scalars().all()
96
+
97
+ @classmethod
98
+ def select_enabled_entries(cls):
99
+ with db_sessionmaker() as session, session.begin():
100
+ stmt = select(cls).where(cls.enabled)
101
+ return (session.execute(stmt)).scalars().all()
102
+
103
+ @classmethod
104
+ def delete(cls, tasks: list[str]):
105
+ with db_sessionmaker() as session, session.begin():
106
+ stmt = delete(cls).where(cls.task.in_(tasks))
107
+ session.execute(stmt)
108
+
109
+ def __repr__(self):
110
+ return (
111
+ f"Task: {self.task} Args: {self.args} Kwargs: {self.kwargs} Schedule: {self.schedule}"
112
+ )
113
+
114
+
115
+ @event.listens_for(CeleryTasksScheduleModel, "before_insert")
116
+ def before_insert_handler(mapper, connection, target: CeleryTasksScheduleModel):
117
+ before_validator(mapper, connection, target)
118
+
119
+
120
+ @event.listens_for(CeleryTasksScheduleModel, "before_update")
121
+ def before_update_handler(mapper, connection, target: CeleryTasksScheduleModel):
122
+ before_validator(mapper, connection, target)
123
+
124
+
125
+ @event.listens_for(CeleryTasksScheduleModel, "after_insert")
126
+ def after_insert_handler(mapper, connection, target: CeleryTasksScheduleModel):
127
+ CeleryTasksScheduleMetaModel.update_last_updated_at()
128
+
129
+
130
+ @event.listens_for(CeleryTasksScheduleModel, "after_update")
131
+ def after_update_handler(mapper, connection, target: CeleryTasksScheduleModel):
132
+ CeleryTasksScheduleMetaModel.update_last_updated_at()
133
+
134
+
135
+ @event.listens_for(CeleryTasksScheduleModel, "after_delete")
136
+ def after_delete_handler(mapper, connection, target: CeleryTasksScheduleModel):
137
+ CeleryTasksScheduleMetaModel.update_last_updated_at()
138
+
139
+
140
+ def before_validator(mapper, connection, target):
141
+ if target.task not in celery_app.tasks:
142
+ raise RuntimeError(f"Task '{target.task}' does not exist")
143
+
144
+ try:
145
+ args = json.loads(target.args)
146
+ except JSONDecodeError as e:
147
+ raise RuntimeError(f"Invalid format for task positional arguments: {e}")
148
+
149
+ try:
150
+ kwargs = json.loads(target.kwargs)
151
+ except JSONDecodeError as e:
152
+ raise RuntimeError(f"Invalid format for task keyword arguments: {e}")
153
+
154
+ task = celery_app.tasks[target.task]
155
+ try:
156
+ check_arguments = task.__header__
157
+ except AttributeError: # pragma: no cover
158
+ debug(f"Task '{task}' has no attribute '__header__': cannot validate arguments")
159
+ else:
160
+ check_arguments(*(args or ()), **(kwargs or {}))
161
+
162
+ try:
163
+ minute, hour, day_of_month, month_of_year, day_of_week = target.schedule.split()
164
+ crontab(
165
+ minute=minute,
166
+ hour=hour,
167
+ day_of_week=day_of_week,
168
+ day_of_month=day_of_month,
169
+ month_of_year=month_of_year,
170
+ )
171
+ except Exception as e:
172
+ raise RuntimeError(f"Invalid schedule: {e}")
173
+
174
+ # Generate unique task key
175
+ target.task_key = get_task_key(target.task, args, kwargs)
@@ -0,0 +1,46 @@
1
+ import datetime as dt
2
+
3
+ from sqlalchemy import (
4
+ BigInteger,
5
+ Column,
6
+ DateTime,
7
+ func,
8
+ insert,
9
+ select,
10
+ update,
11
+ )
12
+
13
+ from ..db import db_sessionmaker
14
+ from .base import CeleryTasksScheduleBase
15
+
16
+
17
+ class CeleryTasksScheduleMetaModel(CeleryTasksScheduleBase):
18
+ """Keeps time when celery_tasks_schedule table was last updated."""
19
+
20
+ __tablename__ = "celery_tasks_schedule_meta"
21
+
22
+ id = Column(BigInteger, primary_key=True)
23
+ last_updated_at = Column(DateTime(timezone=True), nullable=False, server_default=func.now())
24
+
25
+ @classmethod
26
+ def init_last_updated_at(cls):
27
+ with db_sessionmaker() as session, session.begin():
28
+ entry = (session.execute(select(cls).where(cls.id == 1))).scalars().first()
29
+ if not entry:
30
+ stmt = insert(cls).values(id=1, last_updated_at=dt.datetime.now(tz=dt.UTC))
31
+ session.execute(stmt)
32
+
33
+ @classmethod
34
+ def update_last_updated_at(cls):
35
+ with db_sessionmaker() as session, session.begin():
36
+ stmt = update(cls).where(cls.id == 1).values(last_updated_at=dt.datetime.now(tz=dt.UTC))
37
+ session.execute(stmt)
38
+
39
+ @classmethod
40
+ def get_last_updated_at(cls) -> dt.datetime:
41
+ with db_sessionmaker() as session, session.begin():
42
+ stmt = select(cls.last_updated_at).where(cls.id == 1)
43
+ return (session.execute(stmt)).scalars().first()
44
+
45
+ def __repr__(self):
46
+ return f"{self.last_updated_at}"
@@ -0,0 +1,221 @@
1
+ import datetime as dt
2
+ import enum
3
+ import inspect
4
+ import json
5
+ import time
6
+
7
+ from celery.beat import ScheduleEntry, Scheduler, debug, error
8
+ from celery.schedules import crontab
9
+ from celery.utils.time import get_exponential_backoff_interval
10
+ from sqlalchemy.exc import DatabaseError
11
+
12
+ from .db import engine
13
+ from .models.base import CeleryTasksScheduleBase
14
+ from .models.celery_tasks import CeleryTasksModel
15
+ from .models.celery_tasks_schedule import CeleryTasksScheduleModel
16
+ from .models.celery_tasks_schedule_meta import CeleryTasksScheduleMetaModel
17
+ from .schemas.db.celery_tasks import CeleryTasksDbSchema
18
+ from .schemas.db.celery_tasks_schedule import CeleryTasksScheduleDbSchema
19
+ from .utils import get_task_category, get_task_key
20
+
21
+ DEFAULT_MAX_LOOP_INTERVAL = 5 # seconds
22
+ MAX_SYNC_SCHEDULE_INTERVAL = 5 # minutes
23
+ PREPARE_MODELS_MAX_RETRIES = 10
24
+
25
+
26
+ class DatabaseScheduler(Scheduler):
27
+ """Database-backed Beat Scheduler."""
28
+
29
+ ModelTask = CeleryTasksModel
30
+ ModelSchedule = CeleryTasksScheduleModel
31
+ ModelMeta = CeleryTasksScheduleMetaModel
32
+
33
+ _schedule = None
34
+ _last_updated_at = None
35
+
36
+ def __init__(self, *args, **kwargs):
37
+ """Initialize the database scheduler."""
38
+ Scheduler.__init__(self, *args, **kwargs)
39
+ self.max_interval = (
40
+ kwargs.get("max_interval")
41
+ or self.app.conf.beat_max_loop_interval
42
+ or DEFAULT_MAX_LOOP_INTERVAL
43
+ )
44
+
45
+ @property
46
+ def schedule(self):
47
+ self._schedule = self._schedule or {}
48
+ if self._need_update():
49
+ self._update_schedule()
50
+ return self._schedule
51
+
52
+ def setup_schedule(self):
53
+ debug("Setup schedule")
54
+ self._prepare_models()
55
+ self._clean_deprecated()
56
+ self._fill_celery_tasks()
57
+ self._fill_celery_tasks_schedule()
58
+ self._update_schedule()
59
+
60
+ def _fill_celery_tasks(self):
61
+ """Fill celery_tasks table with application celery tasks."""
62
+
63
+ def convert_type(param_type) -> str:
64
+ if param_type is inspect._empty:
65
+ return ""
66
+ if type(param_type) is type:
67
+ return param_type.__name__
68
+ if type(param_type) is enum.EnumType:
69
+ return f"<one of: {", ".join(set(st.value for st in param_type))}>"
70
+ return str(param_type)
71
+
72
+ def get_annotation(param_annotation) -> str:
73
+ if param_annotation is inspect._empty:
74
+ return ""
75
+ return f": {convert_type(param_annotation)}"
76
+
77
+ def get_default(param_default) -> str:
78
+ if param_default is inspect._empty:
79
+ return ""
80
+ return f" [default: {convert_type(param_default)}]"
81
+
82
+ if self.app.tasks:
83
+ db_entries = [
84
+ CeleryTasksDbSchema(
85
+ task=task_name,
86
+ params=", ".join(
87
+ [
88
+ f"{name}{get_annotation(param.annotation)}{get_default(param.default)}"
89
+ for name, param in inspect.signature(task).parameters.items()
90
+ if name != "kwargs"
91
+ ]
92
+ ),
93
+ description=inspect.getdoc(task) if task.__doc__ else "",
94
+ tags=get_task_category(task.__module__),
95
+ )
96
+ for task_name, task in sorted(self.app.tasks.items(), key=lambda item: item[0])
97
+ if not task_name.startswith("celery.")
98
+ ]
99
+ self.ModelTask.bulk_insert(data=db_entries)
100
+ self.ModelTask.bulk_update(data=db_entries)
101
+
102
+ def _fill_celery_tasks_schedule(self):
103
+ """Fill celery_tasks_schedule table with application beat schedule."""
104
+ if self.app.conf.beat_schedule:
105
+ comment = (
106
+ f"loaded from app celery beat schedule on {dt.datetime.now(tz=dt.UTC).isoformat()}"
107
+ )
108
+ db_entries = [
109
+ CeleryTasksScheduleDbSchema(
110
+ task_key=get_task_key(task["task"], task["args"], task["kwargs"]),
111
+ task=task["task"],
112
+ args=json.dumps(task["args"]),
113
+ kwargs=json.dumps(task["kwargs"]),
114
+ schedule=(
115
+ f"{task["schedule"]._orig_minute} "
116
+ f"{task["schedule"]._orig_hour} "
117
+ f"{task["schedule"]._orig_day_of_month} "
118
+ f"{task["schedule"]._orig_month_of_year} "
119
+ f"{task["schedule"]._orig_day_of_week}"
120
+ ),
121
+ comment=comment,
122
+ )
123
+ for task in self.app.conf.beat_schedule.values()
124
+ ]
125
+ self.ModelSchedule.bulk_insert(data=db_entries)
126
+
127
+ def update_from_dict(self, dict_):
128
+ self._schedule = self._schedule or {}
129
+ self._schedule.update(
130
+ {name: self._maybe_entry(name, entry) for name, entry in dict_.items()}
131
+ )
132
+
133
+ def _need_update(self) -> bool:
134
+ now = dt.datetime.now(tz=dt.UTC)
135
+
136
+ if self.max_interval < 30: # noqa
137
+ # Don't update at the beginning and at the end of a minute
138
+ # to avoid overriding current heap
139
+ if 0 <= now.second < 20 or now.second > 50: # noqa
140
+ return False
141
+
142
+ # Sync beat schedule with db every 5 minutes
143
+ # in case we didn't get sqlalchemy update event
144
+ sync_threshold = now - dt.timedelta(minutes=MAX_SYNC_SCHEDULE_INTERVAL)
145
+
146
+ if ( # noqa
147
+ self._last_updated_at is None
148
+ or self._last_updated_at < sync_threshold
149
+ or self._last_updated_at < self.ModelMeta.get_last_updated_at()
150
+ ):
151
+ return True
152
+ return False
153
+
154
+ def _update_schedule(self):
155
+ enabled_tasks = self._get_enabled_tasks()
156
+ self._schedule = enabled_tasks
157
+ self._last_updated_at = dt.datetime.now(tz=dt.UTC)
158
+ self.install_default_entries(self._schedule)
159
+ debug("Current schedule:\n" + "\n".join(repr(entry) for entry in self._schedule.values()))
160
+
161
+ def _get_enabled_tasks(self) -> dict[str, ScheduleEntry]:
162
+ """Return list of enabled periodic tasks."""
163
+ rows = self.ModelSchedule.select_enabled_entries()
164
+ entries = {}
165
+ for row in rows:
166
+ try:
167
+ entries[row.task_key] = self._model_to_entry(model=row)
168
+ except Exception as e:
169
+ error(str(e))
170
+ return entries
171
+
172
+ def _model_to_entry(self, model: CeleryTasksScheduleModel) -> ScheduleEntry:
173
+ args = json.loads(model.args)
174
+ kwargs = json.loads(model.kwargs)
175
+ minute, hour, day_of_month, month_of_year, day_of_week = model.schedule.split()
176
+ schedule = crontab(
177
+ minute=minute,
178
+ hour=hour,
179
+ day_of_week=day_of_week,
180
+ day_of_month=day_of_month,
181
+ month_of_year=month_of_year,
182
+ )
183
+ entry_dict = {
184
+ "task": model.task,
185
+ "args": args,
186
+ "kwargs": kwargs,
187
+ "schedule": schedule,
188
+ }
189
+ return self._maybe_entry(model.task_key, entry_dict)
190
+
191
+ def _prepare_models(self):
192
+ # ###
193
+ # COPIED from: celery.backends.database.session.SessionManager.prepare_models
194
+ # ###
195
+ # SQLAlchemy will check if the items exist before trying to
196
+ # create them, which is a race condition. If it raises an error
197
+ # in one iteration, the next may pass all the existence checks
198
+ # and the call will succeed.
199
+ retries = 0
200
+ while True:
201
+ try:
202
+ CeleryTasksScheduleBase.metadata.create_all(engine)
203
+ except DatabaseError:
204
+ if retries < PREPARE_MODELS_MAX_RETRIES:
205
+ sleep_amount_ms = get_exponential_backoff_interval(10, retries, 1000, True)
206
+ time.sleep(sleep_amount_ms / 1000)
207
+ retries += 1
208
+ else:
209
+ raise
210
+ else:
211
+ break
212
+
213
+ self.ModelMeta.init_last_updated_at()
214
+
215
+ def _clean_deprecated(self):
216
+ """Sync celery_tasks table with application tasks."""
217
+ tasks = set(self.app.tasks)
218
+ db_tasks = set(self.ModelTask.select_all_tasks())
219
+ deprecated_tasks = db_tasks - tasks
220
+ if deprecated_tasks:
221
+ self.ModelTask.delete(tasks=list(deprecated_tasks))
@@ -0,0 +1,18 @@
1
+ from dataclasses import dataclass, fields
2
+
3
+ from sqlalchemy import bindparam
4
+
5
+ from ...utils import db_bindparam
6
+
7
+
8
+ @dataclass
9
+ class BindparamDbSchema:
10
+ @classmethod
11
+ def get_bindparams(cls, exclude: list[str] | None = None):
12
+ exclude = exclude or []
13
+ return {
14
+ f.name: bindparam(db_bindparam(f.name)) for f in fields(cls) if f.name not in exclude
15
+ }
16
+
17
+ def model_dump(self):
18
+ return {db_bindparam(f.name): getattr(self, f.name) for f in fields(self)}
@@ -0,0 +1,11 @@
1
+ from dataclasses import dataclass
2
+
3
+ from .base import BindparamDbSchema
4
+
5
+
6
+ @dataclass
7
+ class CeleryTasksDbSchema(BindparamDbSchema):
8
+ task: str
9
+ params: str
10
+ description: str
11
+ tags: str
@@ -0,0 +1,14 @@
1
+ from dataclasses import dataclass
2
+
3
+ from .base import BindparamDbSchema
4
+
5
+
6
+ @dataclass
7
+ class CeleryTasksScheduleDbSchema(BindparamDbSchema):
8
+ task_key: str
9
+ task: str
10
+ args: str
11
+ kwargs: str
12
+ schedule: str
13
+ enabled: bool = True
14
+ comment: str | None = None
@@ -0,0 +1,26 @@
1
+ import re
2
+
3
+
4
+ def get_task_key(name, args, kwargs) -> str:
5
+ """
6
+ Get task unique identifier.
7
+ The same task can be run with different args/kwargs, and they are different entries for celery beat.
8
+ To uniquely identify those entries we use combination of task_name-args-kwargs.
9
+ """
10
+ args = "-".join(map(str, args))
11
+ kwargs = "-".join([f"{k}-{v}" for k, v in kwargs.items()])
12
+ # Zabbix compatible key format: 0-9a-zA-Z_-.
13
+ return re.sub(r"[\[\](){}, '\"]", "", f"{name}-{args}-{kwargs}".strip("-"))
14
+
15
+
16
+ def get_task_category(task_module: str):
17
+ """
18
+ Get task category by its module:
19
+ src.apps.users.tasks -> users;
20
+ src.apps.base.tasks.tasks -> base.
21
+ """
22
+ return task_module.replace(".tasks", "").split(".")[-1]
23
+
24
+
25
+ def db_bindparam(name: str) -> str:
26
+ return f"{name}_value"