django-stack 0.1.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {django_stack-0.1.0 → django_stack-0.3.0}/PKG-INFO +1 -1
- {django_stack-0.1.0 → django_stack-0.3.0}/pyproject.toml +1 -1
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/__init__.py +1 -1
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/backup.py +89 -20
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/conf.py +2 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/backup_database.py +16 -6
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/restore_database.py +12 -2
- django_stack-0.3.0/src/django_stack/management/commands/restore_media.py +28 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/run_backups.py +9 -1
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/settings.py +4 -1
- django_stack-0.3.0/src/django_stack/tasks.py +83 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_backup.py +60 -0
- django_stack-0.3.0/tests/test_tasks.py +49 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/uv.lock +1 -1
- {django_stack-0.1.0 → django_stack-0.3.0}/.gitignore +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/LICENSE +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/README.md +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/apps.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/checks.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/health/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/health/urls.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/health/views.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/revalidate/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/revalidate/receivers.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/contrib/revalidate/tasks.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/env.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/create_admin.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/create_bucket.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/export_content.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/import_content.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/seed_content.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/wait_for_db.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/models.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/preview.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/rest/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/rest/filters.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/rest/permissions.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/rest/serializers.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/rest/throttling.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/signing.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/staticfiles.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/studio.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/urls.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/conftest.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/settings.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_api.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_env.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_settings.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_signing.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_staticfiles.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_urls.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/test_version.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/testapp/__init__.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/testapp/apps.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/testapp/models.py +0 -0
- {django_stack-0.1.0 → django_stack-0.3.0}/tests/urls.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: django-stack
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: The backend glue of DjangoStack: settings from environment variables, API defaults, health checks and system checks.
|
|
5
5
|
Project-URL: Homepage, https://github.com/dwarsbit/django-stack
|
|
6
6
|
Project-URL: Source, https://github.com/dwarsbit/django-stack/tree/main/python/django-stack
|
|
@@ -1,10 +1,12 @@
|
|
|
1
1
|
"""
|
|
2
|
-
|
|
3
|
-
|
|
2
|
+
Backups: the database with ``pg_dump`` and, when uploads are stored on
|
|
3
|
+
disk, the media directory as a ``.tar.gz``, to a directory and, optionally,
|
|
4
|
+
to a private S3 bucket, with retention.
|
|
4
5
|
|
|
5
6
|
The ``backup`` service of a project runs ``run_backups``, which calls
|
|
6
|
-
:func:`create_backup` once a day.
|
|
7
|
-
``restore_database``
|
|
7
|
+
:func:`create_backup` and :func:`create_media_backup` once a day.
|
|
8
|
+
``backup_database`` makes them now, ``restore_database`` and
|
|
9
|
+
``restore_media`` restore them.
|
|
8
10
|
"""
|
|
9
11
|
|
|
10
12
|
from __future__ import annotations
|
|
@@ -13,6 +15,7 @@ import os
|
|
|
13
15
|
import re
|
|
14
16
|
import shutil
|
|
15
17
|
import subprocess
|
|
18
|
+
import tarfile
|
|
16
19
|
from dataclasses import dataclass
|
|
17
20
|
from datetime import UTC, datetime
|
|
18
21
|
from pathlib import Path
|
|
@@ -23,6 +26,8 @@ from django.core.exceptions import ImproperlyConfigured
|
|
|
23
26
|
from . import conf
|
|
24
27
|
|
|
25
28
|
NAME_PATTERN = re.compile(r"^(?P<prefix>.+)-(?P<stamp>\d{8}T\d{6}Z)\.dump$")
|
|
29
|
+
MEDIA_PATTERN = re.compile(r"^(?P<prefix>.+)-(?P<stamp>\d{8}T\d{6}Z)\.media\.tar\.gz$")
|
|
30
|
+
MEDIA_SUFFIX = ".media.tar.gz"
|
|
26
31
|
|
|
27
32
|
|
|
28
33
|
@dataclass
|
|
@@ -69,6 +74,11 @@ def backup_name(now: datetime | None = None) -> str:
|
|
|
69
74
|
return f"{prefix}-{now.strftime('%Y%m%dT%H%M%SZ')}.dump"
|
|
70
75
|
|
|
71
76
|
|
|
77
|
+
def media_name(database_name: str) -> str:
|
|
78
|
+
"""The media archive that belongs to a database dump: same name and time."""
|
|
79
|
+
return database_name.removesuffix(".dump") + MEDIA_SUFFIX
|
|
80
|
+
|
|
81
|
+
|
|
72
82
|
def s3_client():
|
|
73
83
|
"""A boto3 client for the backup bucket, with the S3_* credentials."""
|
|
74
84
|
import boto3
|
|
@@ -91,13 +101,13 @@ def _s3_target() -> tuple[str, str] | None:
|
|
|
91
101
|
return bucket, f"{prefix}/" if prefix else ""
|
|
92
102
|
|
|
93
103
|
|
|
94
|
-
def create_backup(*, keep: int | None = None) -> Backup:
|
|
104
|
+
def create_backup(*, keep: int | None = None, name: str | None = None) -> Backup:
|
|
95
105
|
"""Dump the database, upload it when a backup bucket is set, and apply the retention."""
|
|
96
106
|
pg_dump = shutil.which("pg_dump")
|
|
97
107
|
if pg_dump is None:
|
|
98
108
|
raise ImproperlyConfigured("pg_dump is not installed; install the Postgres client.")
|
|
99
109
|
directory = backup_dir()
|
|
100
|
-
name = backup_name()
|
|
110
|
+
name = name or backup_name()
|
|
101
111
|
path = directory / name
|
|
102
112
|
partial = path.with_suffix(".partial")
|
|
103
113
|
subprocess.run( # noqa: S603 (fixed arguments, no shell)
|
|
@@ -110,28 +120,60 @@ def create_backup(*, keep: int | None = None) -> Backup:
|
|
|
110
120
|
path.chmod(0o600)
|
|
111
121
|
backup = Backup(name=name, path=path)
|
|
112
122
|
|
|
123
|
+
_upload(backup)
|
|
124
|
+
prune(keep if keep is not None else conf.get("BACKUP_KEEP"))
|
|
125
|
+
return backup
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def _upload(backup: Backup) -> None:
|
|
113
129
|
target = _s3_target()
|
|
114
130
|
if target:
|
|
115
131
|
bucket, prefix = target
|
|
116
|
-
s3_client().upload_file(str(path), bucket, f"{prefix}{name}")
|
|
132
|
+
s3_client().upload_file(str(backup.path), bucket, f"{prefix}{backup.name}")
|
|
117
133
|
backup.uploaded = True
|
|
118
134
|
|
|
119
|
-
|
|
135
|
+
|
|
136
|
+
def media_on_disk() -> bool:
|
|
137
|
+
"""Whether uploads are stored in MEDIA_ROOT, not in S3: only then the media needs a backup."""
|
|
138
|
+
backend = settings.STORAGES.get("default", {}).get("BACKEND", "")
|
|
139
|
+
return backend == "django.core.files.storage.FileSystemStorage"
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def media_backups_enabled() -> bool:
|
|
143
|
+
return bool(conf.get("BACKUP_MEDIA")) and media_on_disk()
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def create_media_backup(database_name: str, *, keep: int | None = None) -> Backup:
|
|
147
|
+
"""Archive MEDIA_ROOT next to a database dump, upload it, and apply the retention."""
|
|
148
|
+
root = Path(settings.MEDIA_ROOT)
|
|
149
|
+
name = media_name(database_name)
|
|
150
|
+
path = backup_dir() / name
|
|
151
|
+
partial = path.with_name(f"{name}.partial")
|
|
152
|
+
with tarfile.open(partial, "w:gz") as archive:
|
|
153
|
+
if root.is_dir():
|
|
154
|
+
for entry in sorted(root.iterdir()):
|
|
155
|
+
if not entry.name.startswith(".restoring"):
|
|
156
|
+
archive.add(entry, arcname=entry.name)
|
|
157
|
+
partial.rename(path)
|
|
158
|
+
path.chmod(0o600)
|
|
159
|
+
backup = Backup(name=name, path=path)
|
|
160
|
+
_upload(backup)
|
|
161
|
+
prune(keep if keep is not None else conf.get("BACKUP_KEEP"), pattern=MEDIA_PATTERN)
|
|
120
162
|
return backup
|
|
121
163
|
|
|
122
164
|
|
|
123
|
-
def _sorted_names(names) -> list[str]:
|
|
165
|
+
def _sorted_names(names, pattern: re.Pattern = NAME_PATTERN) -> list[str]:
|
|
124
166
|
return sorted(
|
|
125
|
-
(name for name in names if
|
|
126
|
-
key=lambda name:
|
|
167
|
+
(name for name in names if pattern.match(name)),
|
|
168
|
+
key=lambda name: pattern.match(name)["stamp"],
|
|
127
169
|
)
|
|
128
170
|
|
|
129
171
|
|
|
130
|
-
def local_backups() -> list[str]:
|
|
131
|
-
return _sorted_names(path.name for path in backup_dir().iterdir())
|
|
172
|
+
def local_backups(pattern: re.Pattern = NAME_PATTERN) -> list[str]:
|
|
173
|
+
return _sorted_names((path.name for path in backup_dir().iterdir()), pattern)
|
|
132
174
|
|
|
133
175
|
|
|
134
|
-
def remote_backups() -> list[str]:
|
|
176
|
+
def remote_backups(pattern: re.Pattern = NAME_PATTERN) -> list[str]:
|
|
135
177
|
target = _s3_target()
|
|
136
178
|
if not target:
|
|
137
179
|
return []
|
|
@@ -139,22 +181,22 @@ def remote_backups() -> list[str]:
|
|
|
139
181
|
names = []
|
|
140
182
|
for page in s3_client().get_paginator("list_objects_v2").paginate(Bucket=bucket, Prefix=prefix):
|
|
141
183
|
names += [item["Key"][len(prefix) :] for item in page.get("Contents", [])]
|
|
142
|
-
return _sorted_names(names)
|
|
184
|
+
return _sorted_names(names, pattern)
|
|
143
185
|
|
|
144
186
|
|
|
145
|
-
def prune(keep: int) -> list[str]:
|
|
146
|
-
"""Delete all but the ``keep`` newest backups, locally and in the bucket."""
|
|
187
|
+
def prune(keep: int, pattern: re.Pattern = NAME_PATTERN) -> list[str]:
|
|
188
|
+
"""Delete all but the ``keep`` newest backups of a kind, locally and in the bucket."""
|
|
147
189
|
removed = []
|
|
148
190
|
if keep is None or keep <= 0:
|
|
149
191
|
return removed
|
|
150
|
-
for name in local_backups()[:-keep]:
|
|
192
|
+
for name in local_backups(pattern)[:-keep]:
|
|
151
193
|
(backup_dir() / name).unlink(missing_ok=True)
|
|
152
194
|
removed.append(name)
|
|
153
195
|
target = _s3_target()
|
|
154
196
|
if target:
|
|
155
197
|
bucket, prefix = target
|
|
156
198
|
client = s3_client()
|
|
157
|
-
for name in remote_backups()[:-keep]:
|
|
199
|
+
for name in remote_backups(pattern)[:-keep]:
|
|
158
200
|
client.delete_object(Bucket=bucket, Key=f"{prefix}{name}")
|
|
159
201
|
removed.append(f"s3://{bucket}/{prefix}{name}")
|
|
160
202
|
return removed
|
|
@@ -169,7 +211,7 @@ def fetch_backup(name: str) -> Path:
|
|
|
169
211
|
if local.is_file():
|
|
170
212
|
return local
|
|
171
213
|
target = _s3_target()
|
|
172
|
-
if target and NAME_PATTERN.match(path.name):
|
|
214
|
+
if target and (NAME_PATTERN.match(path.name) or MEDIA_PATTERN.match(path.name)):
|
|
173
215
|
bucket, prefix = target
|
|
174
216
|
s3_client().download_file(bucket, f"{prefix}{path.name}", str(local))
|
|
175
217
|
return local
|
|
@@ -197,3 +239,30 @@ def restore_backup(path: Path) -> None:
|
|
|
197
239
|
check=True,
|
|
198
240
|
capture_output=True,
|
|
199
241
|
)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def restore_media(path: Path) -> None:
|
|
245
|
+
"""Replace the contents of MEDIA_ROOT with a media archive."""
|
|
246
|
+
root = Path(settings.MEDIA_ROOT)
|
|
247
|
+
root.mkdir(parents=True, exist_ok=True)
|
|
248
|
+
# Extract next to the current files first, so a broken archive changes nothing.
|
|
249
|
+
staging = root / ".restoring"
|
|
250
|
+
if staging.exists():
|
|
251
|
+
shutil.rmtree(staging)
|
|
252
|
+
staging.mkdir()
|
|
253
|
+
try:
|
|
254
|
+
with tarfile.open(path, "r:gz") as archive:
|
|
255
|
+
archive.extractall(staging, filter="data")
|
|
256
|
+
except Exception:
|
|
257
|
+
shutil.rmtree(staging, ignore_errors=True)
|
|
258
|
+
raise
|
|
259
|
+
for entry in root.iterdir():
|
|
260
|
+
if entry == staging:
|
|
261
|
+
continue
|
|
262
|
+
if entry.is_dir() and not entry.is_symlink():
|
|
263
|
+
shutil.rmtree(entry)
|
|
264
|
+
else:
|
|
265
|
+
entry.unlink()
|
|
266
|
+
for entry in staging.iterdir():
|
|
267
|
+
entry.rename(root / entry.name)
|
|
268
|
+
staging.rmdir()
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/backup_database.py
RENAMED
|
@@ -7,18 +7,23 @@ from django_stack import backup
|
|
|
7
7
|
|
|
8
8
|
|
|
9
9
|
class Command(BaseCommand):
|
|
10
|
-
help =
|
|
10
|
+
help = (
|
|
11
|
+
"Back up the database with pg_dump and, when uploads are on disk, the media directory, "
|
|
12
|
+
"to BACKUP_DIR and the backup bucket, and apply the retention."
|
|
13
|
+
)
|
|
11
14
|
|
|
12
15
|
def add_arguments(self, parser):
|
|
13
16
|
parser.add_argument("--keep", type=int, help="How many backups to keep (default: BACKUP_KEEP).")
|
|
14
17
|
parser.add_argument("--list", action="store_true", help="List the backups instead of making one.")
|
|
18
|
+
parser.add_argument("--no-media", action="store_true", help="Back up only the database.")
|
|
15
19
|
|
|
16
|
-
def handle(self, *args, keep=None, list=False, **options):
|
|
20
|
+
def handle(self, *args, keep=None, list=False, no_media=False, **options):
|
|
17
21
|
if list:
|
|
18
|
-
for
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
+
for pattern in (backup.NAME_PATTERN, backup.MEDIA_PATTERN):
|
|
23
|
+
for name in backup.local_backups(pattern):
|
|
24
|
+
self.stdout.write(f"local {name}")
|
|
25
|
+
for name in backup.remote_backups(pattern):
|
|
26
|
+
self.stdout.write(f"bucket {name}")
|
|
22
27
|
return
|
|
23
28
|
try:
|
|
24
29
|
result = backup.create_backup(keep=keep)
|
|
@@ -28,3 +33,8 @@ class Command(BaseCommand):
|
|
|
28
33
|
raise CommandError(f"pg_dump failed: {error.stderr.decode(errors='replace').strip()}") from error
|
|
29
34
|
where = " and the backup bucket" if result.uploaded else ""
|
|
30
35
|
self.stdout.write(self.style.SUCCESS(f"Backed up the database to {result.path}{where}."))
|
|
36
|
+
if no_media or not backup.media_backups_enabled():
|
|
37
|
+
return
|
|
38
|
+
media = backup.create_media_backup(result.name, keep=keep)
|
|
39
|
+
where = " and the backup bucket" if media.uploaded else ""
|
|
40
|
+
self.stdout.write(self.style.SUCCESS(f"Backed up the media to {media.path}{where}."))
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/restore_database.py
RENAMED
|
@@ -12,14 +12,21 @@ class Command(BaseCommand):
|
|
|
12
12
|
def add_arguments(self, parser):
|
|
13
13
|
parser.add_argument("backup", help="A file name from `backup_database --list`, or a path to a dump.")
|
|
14
14
|
parser.add_argument("--yes", action="store_true", help="Don't ask for confirmation.")
|
|
15
|
+
parser.add_argument(
|
|
16
|
+
"--media", action="store_true", help="Also restore the media archive made with this backup."
|
|
17
|
+
)
|
|
15
18
|
|
|
16
|
-
def handle(self, *args, yes=False, **options):
|
|
19
|
+
def handle(self, *args, yes=False, media=False, **options):
|
|
17
20
|
try:
|
|
18
21
|
path = backup.fetch_backup(options["backup"])
|
|
22
|
+
media_path = backup.fetch_backup(backup.media_name(path.name)) if media else None
|
|
19
23
|
except FileNotFoundError as error:
|
|
20
24
|
raise CommandError(str(error)) from error
|
|
25
|
+
if media and not backup.media_on_disk():
|
|
26
|
+
raise CommandError("Uploads are stored in S3, not on disk; restore them from the bucket.")
|
|
21
27
|
if not yes:
|
|
22
|
-
|
|
28
|
+
what = f"{path.name} and its media" if media_path else path.name
|
|
29
|
+
answer = input(f"Restore {what}? Content saved since the backup is lost. Type 'yes': ")
|
|
23
30
|
if answer.strip().lower() != "yes":
|
|
24
31
|
raise CommandError("Not restored.")
|
|
25
32
|
try:
|
|
@@ -31,3 +38,6 @@ class Command(BaseCommand):
|
|
|
31
38
|
f"pg_restore failed: {error.stderr.decode(errors='replace').strip()}"
|
|
32
39
|
) from error
|
|
33
40
|
self.stdout.write(self.style.SUCCESS(f"Restored {path.name}."))
|
|
41
|
+
if media_path:
|
|
42
|
+
backup.restore_media(media_path)
|
|
43
|
+
self.stdout.write(self.style.SUCCESS(f"Restored the media from {media_path.name}."))
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
from django.core.management.base import BaseCommand, CommandError
|
|
2
|
+
|
|
3
|
+
from django_stack import backup
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Command(BaseCommand):
|
|
7
|
+
help = "Replace the media directory with an archive made by backup_database."
|
|
8
|
+
|
|
9
|
+
def add_arguments(self, parser):
|
|
10
|
+
parser.add_argument("backup", help="A .media.tar.gz name from `backup_database --list`, or a path.")
|
|
11
|
+
parser.add_argument("--yes", action="store_true", help="Don't ask for confirmation.")
|
|
12
|
+
|
|
13
|
+
def handle(self, *args, yes=False, **options):
|
|
14
|
+
if not backup.media_on_disk():
|
|
15
|
+
raise CommandError("Uploads are stored in S3, not on disk; restore them from the bucket.")
|
|
16
|
+
try:
|
|
17
|
+
path = backup.fetch_backup(options["backup"])
|
|
18
|
+
except FileNotFoundError as error:
|
|
19
|
+
raise CommandError(str(error)) from error
|
|
20
|
+
if not yes:
|
|
21
|
+
answer = input(f"Replace the media with {path.name}? Files uploaded since are lost. Type 'yes': ")
|
|
22
|
+
if answer.strip().lower() != "yes":
|
|
23
|
+
raise CommandError("Not restored.")
|
|
24
|
+
try:
|
|
25
|
+
backup.restore_media(path)
|
|
26
|
+
except Exception as error:
|
|
27
|
+
raise CommandError(f"Could not restore {path.name}: {error}") from error
|
|
28
|
+
self.stdout.write(self.style.SUCCESS(f"Restored the media from {path.name}."))
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/run_backups.py
RENAMED
|
@@ -21,7 +21,10 @@ def next_run(at: str, now: datetime) -> datetime:
|
|
|
21
21
|
|
|
22
22
|
|
|
23
23
|
class Command(BaseCommand):
|
|
24
|
-
help =
|
|
24
|
+
help = (
|
|
25
|
+
"Back up the database, and the media when it is on disk, every day at BACKUP_TIME (UTC). "
|
|
26
|
+
"The backup service runs this."
|
|
27
|
+
)
|
|
25
28
|
|
|
26
29
|
def handle(self, *args, **options):
|
|
27
30
|
at = conf.get("BACKUP_TIME")
|
|
@@ -37,6 +40,11 @@ class Command(BaseCommand):
|
|
|
37
40
|
result.path,
|
|
38
41
|
" and the bucket" if result.uploaded else "",
|
|
39
42
|
)
|
|
43
|
+
if backup.media_backups_enabled():
|
|
44
|
+
media = backup.create_media_backup(result.name)
|
|
45
|
+
logger.info(
|
|
46
|
+
"Backed up the media to %s%s", media.path, " and the bucket" if media.uploaded else ""
|
|
47
|
+
)
|
|
40
48
|
except subprocess.CalledProcessError as error:
|
|
41
49
|
logger.error("pg_dump failed: %s", error.stderr.decode(errors="replace").strip())
|
|
42
50
|
except Exception:
|
|
@@ -27,7 +27,8 @@ to ``SITE_URL``), ``ALLOWED_HOSTS`` (extra hosts, comma separated),
|
|
|
27
27
|
``API_SECRET_KEY`` (full access), ``API_READ_KEY`` (read-only),
|
|
28
28
|
``PREVIEW_SECRET``, ``REVALIDATE_URL``, ``REVALIDATE_SECRET``,
|
|
29
29
|
``NUM_PROXIES``, ``PROJECT_NAME``, and the backup variables ``BACKUP_DIR``,
|
|
30
|
-
``BACKUP_KEEP``, ``BACKUP_TIME``, ``BACKUP_S3_BUCKET`` and
|
|
30
|
+
``BACKUP_KEEP``, ``BACKUP_TIME``, ``BACKUP_MEDIA``, ``BACKUP_S3_BUCKET`` and
|
|
31
|
+
``BACKUP_S3_PREFIX``.
|
|
31
32
|
"""
|
|
32
33
|
|
|
33
34
|
from __future__ import annotations
|
|
@@ -299,6 +300,8 @@ def configure(
|
|
|
299
300
|
# A private bucket: never the public media bucket.
|
|
300
301
|
"BACKUP_S3_BUCKET": e.str("BACKUP_S3_BUCKET"),
|
|
301
302
|
"BACKUP_S3_PREFIX": e.str("BACKUP_S3_PREFIX", "backups/"),
|
|
303
|
+
"BACKUP_MEDIA": e.bool("BACKUP_MEDIA", True),
|
|
304
|
+
"BACKUP_DAILY": e.bool("BACKUP_DAILY", False),
|
|
302
305
|
"BACKUP_S3": {
|
|
303
306
|
"endpoint_url": e.str("BACKUP_S3_ENDPOINT_URL", e.str("S3_ENDPOINT_URL")),
|
|
304
307
|
"region_name": e.str("BACKUP_S3_REGION", e.str("S3_REGION")),
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
"""
|
|
2
|
+
The daily backup as a Huey periodic task, run by the worker.
|
|
3
|
+
|
|
4
|
+
The task runs every hour at the minute of ``BACKUP_TIME`` (UTC) and backs
|
|
5
|
+
up when it is the hour of ``BACKUP_TIME``, or when the newest backup is
|
|
6
|
+
older than a day: that catches up a backup missed while the server or the
|
|
7
|
+
worker was down. The worker also checks once when it starts. It only runs where
|
|
8
|
+
``BACKUP_DAILY`` is on, which the production worker sets.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import logging
|
|
14
|
+
from datetime import UTC, datetime, timedelta
|
|
15
|
+
|
|
16
|
+
from huey import crontab
|
|
17
|
+
from huey.contrib.djhuey import db_periodic_task, lock_task, on_startup
|
|
18
|
+
|
|
19
|
+
from . import backup, conf
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger("django_stack.backup")
|
|
22
|
+
|
|
23
|
+
#: A backup older than this is due, whatever the time.
|
|
24
|
+
STALE_AFTER = timedelta(hours=25)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def _backup_time() -> tuple[int, int]:
|
|
28
|
+
value = conf.get("BACKUP_TIME") or "03:00"
|
|
29
|
+
try:
|
|
30
|
+
hour, minute = (int(part) for part in value.split(":"))
|
|
31
|
+
except ValueError:
|
|
32
|
+
logger.error("BACKUP_TIME must be HH:MM, not %r; using 03:00.", value)
|
|
33
|
+
return 3, 0
|
|
34
|
+
return hour, minute
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def newest_backup_time() -> datetime | None:
|
|
38
|
+
"""The time of the newest database backup, in the volume or the bucket."""
|
|
39
|
+
names = backup.local_backups()
|
|
40
|
+
try:
|
|
41
|
+
names += backup.remote_backups()
|
|
42
|
+
except Exception:
|
|
43
|
+
logger.exception("Could not list the backup bucket")
|
|
44
|
+
stamps = [backup.NAME_PATTERN.match(name)["stamp"] for name in names]
|
|
45
|
+
if not stamps:
|
|
46
|
+
return None
|
|
47
|
+
return datetime.strptime(max(stamps), "%Y%m%dT%H%M%SZ").replace(tzinfo=UTC)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def backup_due(now: datetime, newest: datetime | None, hour: int) -> bool:
|
|
51
|
+
"""Whether to back up now: at the hour of BACKUP_TIME, or when the newest backup is stale."""
|
|
52
|
+
if newest is None or now - newest >= STALE_AFTER:
|
|
53
|
+
return True
|
|
54
|
+
return now.hour == hour and now - newest > timedelta(hours=1)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def run_daily_backup(now: datetime | None = None) -> bool:
|
|
58
|
+
"""Back up the database and the media when due. Returns whether it backed up."""
|
|
59
|
+
if not conf.get("BACKUP_DAILY"):
|
|
60
|
+
return False
|
|
61
|
+
now = now or datetime.now(UTC)
|
|
62
|
+
hour, _ = _backup_time()
|
|
63
|
+
if not backup_due(now, newest_backup_time(), hour):
|
|
64
|
+
return False
|
|
65
|
+
result = backup.create_backup()
|
|
66
|
+
logger.info("Backed up the database to %s%s", result.path, " and the bucket" if result.uploaded else "")
|
|
67
|
+
if backup.media_backups_enabled():
|
|
68
|
+
media = backup.create_media_backup(result.name)
|
|
69
|
+
logger.info("Backed up the media to %s%s", media.path, " and the bucket" if media.uploaded else "")
|
|
70
|
+
return True
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@db_periodic_task(crontab(minute=str(_backup_time()[1])), retries=3, retry_delay=600)
|
|
74
|
+
@lock_task("django-stack-backup")
|
|
75
|
+
def daily_backup():
|
|
76
|
+
run_daily_backup()
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
@on_startup("django-stack-backup-catch-up")
|
|
80
|
+
def catch_up_backup():
|
|
81
|
+
"""When the worker starts, queue the backup; it only runs when one is due."""
|
|
82
|
+
if conf.get("BACKUP_DAILY"):
|
|
83
|
+
daily_backup()
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import subprocess
|
|
2
|
+
import tarfile
|
|
2
3
|
from datetime import UTC, datetime
|
|
3
4
|
|
|
4
5
|
import pytest
|
|
@@ -83,3 +84,62 @@ def test_next_run():
|
|
|
83
84
|
now = datetime(2026, 11, 4, 2, 30, tzinfo=UTC)
|
|
84
85
|
assert next_run("03:00", now) == datetime(2026, 11, 4, 3, 0, tzinfo=UTC)
|
|
85
86
|
assert next_run("02:00", now) == datetime(2026, 11, 5, 2, 0, tzinfo=UTC)
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@pytest.fixture
|
|
90
|
+
def media(tmp_path, settings, backups):
|
|
91
|
+
root = tmp_path / "media-root"
|
|
92
|
+
(root / "images").mkdir(parents=True)
|
|
93
|
+
(root / "images" / "a.jpg").write_bytes(b"a")
|
|
94
|
+
(root / "b.pdf").write_bytes(b"b")
|
|
95
|
+
settings.MEDIA_ROOT = str(root)
|
|
96
|
+
disk = {"BACKEND": "django.core.files.storage.FileSystemStorage"}
|
|
97
|
+
settings.STORAGES = {**settings.STORAGES, "default": disk}
|
|
98
|
+
return root
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def test_media_name():
|
|
102
|
+
assert backup.media_name("site-20261104T030000Z.dump") == "site-20261104T030000Z.media.tar.gz"
|
|
103
|
+
|
|
104
|
+
|
|
105
|
+
def test_media_backup_and_restore(media, backups):
|
|
106
|
+
result = backup.create_media_backup("site-20261104T030000Z.dump", keep=5)
|
|
107
|
+
assert result.path == backups / "site-20261104T030000Z.media.tar.gz"
|
|
108
|
+
assert backup.local_backups(backup.MEDIA_PATTERN) == [result.name]
|
|
109
|
+
assert backup.local_backups() == []
|
|
110
|
+
|
|
111
|
+
(media / "b.pdf").unlink()
|
|
112
|
+
(media / "new.txt").write_bytes(b"new")
|
|
113
|
+
backup.restore_media(backup.fetch_backup(result.name))
|
|
114
|
+
files = sorted(str(path.relative_to(media)) for path in media.rglob("*"))
|
|
115
|
+
assert files == ["b.pdf", "images", "images/a.jpg"]
|
|
116
|
+
assert (media / "images" / "a.jpg").read_bytes() == b"a"
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def test_broken_media_archive_changes_nothing(media, tmp_path):
|
|
120
|
+
broken = tmp_path / "broken.media.tar.gz"
|
|
121
|
+
broken.write_bytes(b"not a tar")
|
|
122
|
+
with pytest.raises(tarfile.ReadError):
|
|
123
|
+
backup.restore_media(broken)
|
|
124
|
+
assert (media / "b.pdf").exists()
|
|
125
|
+
assert not (media / ".restoring").exists()
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
def test_media_retention_is_separate(media, backups):
|
|
129
|
+
for day in range(1, 4):
|
|
130
|
+
(backups / f"site-2026110{day}T030000Z.dump").write_bytes(b"")
|
|
131
|
+
backup.create_media_backup(f"site-2026110{day}T030000Z.dump", keep=2)
|
|
132
|
+
assert backup.local_backups(backup.MEDIA_PATTERN) == [
|
|
133
|
+
"site-20261102T030000Z.media.tar.gz",
|
|
134
|
+
"site-20261103T030000Z.media.tar.gz",
|
|
135
|
+
]
|
|
136
|
+
assert len(backup.local_backups()) == 3
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def test_media_backups_only_on_disk(media, settings):
|
|
140
|
+
assert backup.media_backups_enabled()
|
|
141
|
+
settings.DJANGO_STACK = {**settings.DJANGO_STACK, "BACKUP_MEDIA": False}
|
|
142
|
+
assert not backup.media_backups_enabled()
|
|
143
|
+
settings.DJANGO_STACK = {**settings.DJANGO_STACK, "BACKUP_MEDIA": True}
|
|
144
|
+
settings.STORAGES = {**settings.STORAGES, "default": {"BACKEND": "storages.backends.s3.S3Storage"}}
|
|
145
|
+
assert not backup.media_backups_enabled()
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
from datetime import UTC, datetime, timedelta
|
|
2
|
+
|
|
3
|
+
import pytest
|
|
4
|
+
|
|
5
|
+
from django_stack import backup, tasks
|
|
6
|
+
|
|
7
|
+
NOW = datetime(2026, 11, 5, 3, 0, tzinfo=UTC)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def test_backup_due_at_the_hour_or_when_stale():
|
|
11
|
+
assert tasks.backup_due(NOW, None, 3)
|
|
12
|
+
assert tasks.backup_due(NOW, NOW - timedelta(hours=24), 3)
|
|
13
|
+
assert not tasks.backup_due(NOW, NOW - timedelta(minutes=30), 3)
|
|
14
|
+
assert not tasks.backup_due(NOW.replace(hour=10), NOW - timedelta(hours=7), 3)
|
|
15
|
+
assert tasks.backup_due(NOW.replace(hour=10), NOW - timedelta(hours=26), 3)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
@pytest.fixture
|
|
19
|
+
def daily(tmp_path, settings, monkeypatch):
|
|
20
|
+
settings.DJANGO_STACK = {
|
|
21
|
+
**settings.DJANGO_STACK,
|
|
22
|
+
"BACKUP_DIR": str(tmp_path),
|
|
23
|
+
"BACKUP_S3_BUCKET": None,
|
|
24
|
+
"BACKUP_DAILY": True,
|
|
25
|
+
"BACKUP_MEDIA": False,
|
|
26
|
+
}
|
|
27
|
+
made = []
|
|
28
|
+
|
|
29
|
+
def fake_backup():
|
|
30
|
+
name = backup.backup_name()
|
|
31
|
+
(tmp_path / name).write_bytes(b"")
|
|
32
|
+
made.append(name)
|
|
33
|
+
return backup.Backup(name=name, path=tmp_path / name)
|
|
34
|
+
|
|
35
|
+
monkeypatch.setattr(backup, "create_backup", fake_backup)
|
|
36
|
+
return made
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_daily_backup_catches_up_and_then_waits(daily):
|
|
40
|
+
assert tasks.run_daily_backup(NOW.replace(hour=10))
|
|
41
|
+
assert len(daily) == 1
|
|
42
|
+
assert not tasks.run_daily_backup(datetime.now(UTC))
|
|
43
|
+
assert len(daily) == 1
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def test_daily_backup_is_off_by_default(daily, settings):
|
|
47
|
+
settings.DJANGO_STACK = {**settings.DJANGO_STACK, "BACKUP_DAILY": False}
|
|
48
|
+
assert not tasks.run_daily_backup(NOW)
|
|
49
|
+
assert daily == []
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/create_admin.py
RENAMED
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/create_bucket.py
RENAMED
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/export_content.py
RENAMED
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/import_content.py
RENAMED
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/seed_content.py
RENAMED
|
File without changes
|
{django_stack-0.1.0 → django_stack-0.3.0}/src/django_stack/management/commands/wait_for_db.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|