django-zerobucket 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- django_zerobucket/__init__.py +24 -0
- django_zerobucket/apps.py +7 -0
- django_zerobucket/management/__init__.py +0 -0
- django_zerobucket/management/commands/__init__.py +0 -0
- django_zerobucket/management/commands/zerobucket_info.py +28 -0
- django_zerobucket/management/commands/zerobucket_tier.py +63 -0
- django_zerobucket/management/commands/zerobucket_verify.py +35 -0
- django_zerobucket/storage.py +179 -0
- django_zerobucket/urls.py +26 -0
- django_zerobucket/views.py +104 -0
- django_zerobucket-0.1.0.dist-info/METADATA +220 -0
- django_zerobucket-0.1.0.dist-info/RECORD +13 -0
- django_zerobucket-0.1.0.dist-info/WHEEL +4 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""django-zerobucket: a Django Storage backend adapter for ZeroBucket.
|
|
2
|
+
|
|
3
|
+
from django_zerobucket import ZeroBucketStorage
|
|
4
|
+
|
|
5
|
+
See storage.py's module docstring for the id-vs-path naming difference
|
|
6
|
+
from typical Django Storage backends, and views.py's for how served
|
|
7
|
+
images work and their access-control scope.
|
|
8
|
+
|
|
9
|
+
default_app_config is not needed (Django's app-loading auto-discovery
|
|
10
|
+
handles apps.py without it since Django 3.2) -- this package still
|
|
11
|
+
needs "django_zerobucket" added to INSTALLED_APPS for its management
|
|
12
|
+
commands (management/commands/zerobucket_*.py) to be discovered, even
|
|
13
|
+
though the Storage backend and serving view themselves would technically
|
|
14
|
+
work without that (Django's STORAGES/urlconf wiring doesn't require
|
|
15
|
+
INSTALLED_APPS membership, only manage.py's command discovery does).
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
from .storage import ZeroBucketStorage
|
|
21
|
+
|
|
22
|
+
__version__ = "0.1.0"
|
|
23
|
+
|
|
24
|
+
__all__ = ["ZeroBucketStorage"]
|
|
File without changes
|
|
File without changes
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
"""./manage.py zerobucket_info -- thin wrapper around zerobucket.cli's
|
|
2
|
+
cmd_info, reusing that exact, already-tested implementation rather than
|
|
3
|
+
reimplementing it. Supplies database_url from the
|
|
4
|
+
ZEROBUCKET_DATABASE_URL Django setting (falling back further to the
|
|
5
|
+
ZEROBUCKET_DATABASE_URL environment variable, same as the standalone
|
|
6
|
+
CLI, since cmd_info's own _resolve_database_url already does that) so
|
|
7
|
+
you don't have to pass --database-url by hand in a Django project that
|
|
8
|
+
already has it configured.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
import argparse
|
|
14
|
+
|
|
15
|
+
from django.conf import settings
|
|
16
|
+
from django.core.management.base import BaseCommand
|
|
17
|
+
from zerobucket.cli import cmd_info
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
class Command(BaseCommand):
|
|
21
|
+
help = "Report ZeroBucket storage stats: image count, total size, format breakdown."
|
|
22
|
+
|
|
23
|
+
def handle(self, *args, **options) -> None:
|
|
24
|
+
cmd_info(
|
|
25
|
+
argparse.Namespace(
|
|
26
|
+
database_url=getattr(settings, "ZEROBUCKET_DATABASE_URL", None)
|
|
27
|
+
)
|
|
28
|
+
)
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
"""./manage.py zerobucket_tier -- thin wrapper around zerobucket.cli's
|
|
2
|
+
cmd_tier. Same argument shape as the standalone `zerobucket tier` CLI
|
|
3
|
+
command (see that command's docs for the full explanation of single-id
|
|
4
|
+
vs bulk selection, --dry-run, and the transactional-safety guarantee of
|
|
5
|
+
tier_to_object_storage() underneath) -- this just supplies
|
|
6
|
+
database_url from Django settings instead of requiring --database-url,
|
|
7
|
+
and requires `pip install zerobucket[s3]` the same way the standalone
|
|
8
|
+
command does (boto3 stays an optional dependency here too, not pulled
|
|
9
|
+
in by installing django-zerobucket itself).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
|
|
16
|
+
from django.conf import settings
|
|
17
|
+
from django.core.management.base import BaseCommand
|
|
18
|
+
from zerobucket.cli import cmd_tier
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Command(BaseCommand):
|
|
22
|
+
help = "Move image(s) from Postgres into S3-compatible object storage."
|
|
23
|
+
|
|
24
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
25
|
+
parser.add_argument(
|
|
26
|
+
"image_id",
|
|
27
|
+
nargs="?",
|
|
28
|
+
default=None,
|
|
29
|
+
help="Tier a single image by id. Omit and use --all/--min-size/"
|
|
30
|
+
"--older-than instead for bulk selection.",
|
|
31
|
+
)
|
|
32
|
+
parser.add_argument(
|
|
33
|
+
"--bucket",
|
|
34
|
+
required=True,
|
|
35
|
+
help="S3(-compatible) bucket -- must already exist.",
|
|
36
|
+
)
|
|
37
|
+
parser.add_argument("--endpoint-url", default=None)
|
|
38
|
+
parser.add_argument("--region", default=None)
|
|
39
|
+
parser.add_argument("--aws-access-key-id", default=None)
|
|
40
|
+
parser.add_argument("--aws-secret-access-key", default=None)
|
|
41
|
+
parser.add_argument("--min-size", type=int, default=None, metavar="BYTES")
|
|
42
|
+
parser.add_argument("--older-than", type=int, default=None, metavar="DAYS")
|
|
43
|
+
parser.add_argument("--all", action="store_true")
|
|
44
|
+
parser.add_argument("--limit", type=int, default=None)
|
|
45
|
+
parser.add_argument("--dry-run", action="store_true")
|
|
46
|
+
|
|
47
|
+
def handle(self, *args, **options) -> None:
|
|
48
|
+
cmd_tier(
|
|
49
|
+
argparse.Namespace(
|
|
50
|
+
database_url=getattr(settings, "ZEROBUCKET_DATABASE_URL", None),
|
|
51
|
+
image_id=options["image_id"],
|
|
52
|
+
bucket=options["bucket"],
|
|
53
|
+
endpoint_url=options["endpoint_url"],
|
|
54
|
+
region=options["region"],
|
|
55
|
+
aws_access_key_id=options["aws_access_key_id"],
|
|
56
|
+
aws_secret_access_key=options["aws_secret_access_key"],
|
|
57
|
+
min_size=options["min_size"],
|
|
58
|
+
older_than=options["older_than"],
|
|
59
|
+
all=options["all"],
|
|
60
|
+
limit=options["limit"],
|
|
61
|
+
dry_run=options["dry_run"],
|
|
62
|
+
)
|
|
63
|
+
)
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"""./manage.py zerobucket_verify -- thin wrapper around
|
|
2
|
+
zerobucket.cli's cmd_verify. Note that cmd_verify calls sys.exit(1) on
|
|
3
|
+
any checksum mismatch (and sys.exit(2) on a bad/missing database URL) --
|
|
4
|
+
that propagates as this management command's process exit code
|
|
5
|
+
unchanged, the same way it would from the standalone CLI, making this
|
|
6
|
+
usable in a cron job or CI step exactly like `zerobucket verify` is.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import argparse
|
|
12
|
+
|
|
13
|
+
from django.conf import settings
|
|
14
|
+
from django.core.management.base import BaseCommand
|
|
15
|
+
from zerobucket.cli import cmd_verify
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class Command(BaseCommand):
|
|
19
|
+
help = "Re-checksum stored images to detect corruption."
|
|
20
|
+
|
|
21
|
+
def add_arguments(self, parser: argparse.ArgumentParser) -> None:
|
|
22
|
+
parser.add_argument(
|
|
23
|
+
"--sample",
|
|
24
|
+
type=int,
|
|
25
|
+
default=None,
|
|
26
|
+
help="Check a random sample of this many images instead of every one.",
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
def handle(self, *args, **options) -> None:
|
|
30
|
+
cmd_verify(
|
|
31
|
+
argparse.Namespace(
|
|
32
|
+
database_url=getattr(settings, "ZEROBUCKET_DATABASE_URL", None),
|
|
33
|
+
sample=options["sample"],
|
|
34
|
+
)
|
|
35
|
+
)
|
|
@@ -0,0 +1,179 @@
|
|
|
1
|
+
"""ZeroBucketStorage: a Django Storage backend adapter, plugging
|
|
2
|
+
ZeroBucket into Django's existing FileField/ImageField via the
|
|
3
|
+
STORAGES setting -- no new field type to learn, works with existing
|
|
4
|
+
forms/admin/migrations the same way django-storages' S3/GCS backends do
|
|
5
|
+
for their targets.
|
|
6
|
+
|
|
7
|
+
# settings.py
|
|
8
|
+
STORAGES = {
|
|
9
|
+
"default": {
|
|
10
|
+
"BACKEND": "django_zerobucket.storage.ZeroBucketStorage",
|
|
11
|
+
"OPTIONS": {"database_url": "postgresql://..."},
|
|
12
|
+
},
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
# models.py
|
|
16
|
+
class Product(models.Model):
|
|
17
|
+
photo = models.ImageField(upload_to="ignored") # see below
|
|
18
|
+
|
|
19
|
+
A DELIBERATE, HONEST DIFFERENCE from typical Django Storage backends,
|
|
20
|
+
worth understanding before adopting this: the `name` your FileField/
|
|
21
|
+
ImageField stores in its database column is the ZeroBucket image_id (a
|
|
22
|
+
UUID), NOT a filesystem-style path built from `upload_to=`. ZeroBucket
|
|
23
|
+
is id-addressed storage, not path-addressed storage -- there's no
|
|
24
|
+
"directory" for an uploaded file to live in, so `upload_to=` is
|
|
25
|
+
effectively ignored (Django still requires the kwarg be present on the
|
|
26
|
+
field, but this backend never uses it to build the stored name). If
|
|
27
|
+
your code reads `instance.photo.name` expecting something like
|
|
28
|
+
"uploads/2024/photo.jpg", it will instead see a UUID. `instance.photo.url`
|
|
29
|
+
and `instance.photo.open()` work exactly like any other Storage backend
|
|
30
|
+
regardless -- this only affects code that inspects the raw name string.
|
|
31
|
+
"""
|
|
32
|
+
|
|
33
|
+
from __future__ import annotations
|
|
34
|
+
|
|
35
|
+
from typing import Any
|
|
36
|
+
|
|
37
|
+
from django.conf import settings
|
|
38
|
+
from django.core.files import File
|
|
39
|
+
from django.core.files.base import ContentFile
|
|
40
|
+
from django.core.files.storage import Storage
|
|
41
|
+
from django.urls import reverse
|
|
42
|
+
from django.utils.deconstruct import deconstructible
|
|
43
|
+
from zerobucket import ZeroBucket
|
|
44
|
+
|
|
45
|
+
_client_cache: dict[tuple, ZeroBucket] = {}
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _get_shared_client(database_url: str, options: dict) -> ZeroBucket:
|
|
49
|
+
"""ZeroBucket instances hold a real connection pool -- constructing
|
|
50
|
+
a new one per Storage instantiation (which Django's own storage
|
|
51
|
+
registry mostly avoids via its own caching, but nothing guarantees
|
|
52
|
+
every caller goes through that registry) would mean leaking pools.
|
|
53
|
+
Cached here, keyed on (database_url, frozen options), so repeated
|
|
54
|
+
ZeroBucketStorage(**same_options) calls -- from Django's registry or
|
|
55
|
+
otherwise, including the serving view constructing its own instance
|
|
56
|
+
(see views.py) -- share one underlying client/pool rather than each
|
|
57
|
+
opening a separate one. Not thread-safety-hardened beyond Python
|
|
58
|
+
dict operations already being atomic under the GIL -- a documented,
|
|
59
|
+
accepted simplicity tradeoff, not a claimed guarantee under free-
|
|
60
|
+
threaded Python.
|
|
61
|
+
"""
|
|
62
|
+
key = (database_url, tuple(sorted(options.items())))
|
|
63
|
+
if key not in _client_cache:
|
|
64
|
+
_client_cache[key] = ZeroBucket(database_url=database_url, **options)
|
|
65
|
+
return _client_cache[key]
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
@deconstructible
|
|
69
|
+
class ZeroBucketStorage(Storage):
|
|
70
|
+
"""See module docstring for the id-vs-path naming difference from
|
|
71
|
+
typical Storage backends before using this.
|
|
72
|
+
|
|
73
|
+
OPTIONS (all optional except database_url, which can instead come
|
|
74
|
+
from the ZEROBUCKET_DATABASE_URL Django setting):
|
|
75
|
+
|
|
76
|
+
database_url: falls back to settings.ZEROBUCKET_DATABASE_URL if
|
|
77
|
+
omitted. Required one way or the other.
|
|
78
|
+
Any other keyword ZeroBucket's own constructor accepts
|
|
79
|
+
(max_bytes, max_pixels, allowed_formats, dedup,
|
|
80
|
+
pool_min_size, pool_max_size, pool_timeout, object_storage,
|
|
81
|
+
before_get, before_put, on_operation) is forwarded as-is --
|
|
82
|
+
this class does not re-document or restrict them.
|
|
83
|
+
|
|
84
|
+
@deconstructible is required for Django migrations to be able to
|
|
85
|
+
serialize a FileField(storage=ZeroBucketStorage(...)) reference --
|
|
86
|
+
without it, `makemigrations` fails on this class specifically. Not
|
|
87
|
+
needed if you only ever configure this via the STORAGES setting
|
|
88
|
+
(Django constructs the instance itself in that case), but included
|
|
89
|
+
unconditionally since supporting the direct storage=... kwarg too
|
|
90
|
+
costs nothing extra.
|
|
91
|
+
"""
|
|
92
|
+
|
|
93
|
+
def __init__(self, database_url: str | None = None, **options: Any) -> None:
|
|
94
|
+
self._database_url = database_url or getattr(
|
|
95
|
+
settings, "ZEROBUCKET_DATABASE_URL", None
|
|
96
|
+
)
|
|
97
|
+
if not self._database_url:
|
|
98
|
+
raise ValueError(
|
|
99
|
+
"ZeroBucketStorage requires database_url (via STORAGES' "
|
|
100
|
+
"OPTIONS) or the ZEROBUCKET_DATABASE_URL Django setting -- "
|
|
101
|
+
"neither was provided."
|
|
102
|
+
)
|
|
103
|
+
self._options = options
|
|
104
|
+
self._client = _get_shared_client(self._database_url, options)
|
|
105
|
+
|
|
106
|
+
# ---- the required Storage overrides ------------------------------
|
|
107
|
+
|
|
108
|
+
def _save(self, name: str, content) -> str:
|
|
109
|
+
"""Ignores `name` as a target location (see module docstring) --
|
|
110
|
+
ZeroBucket always generates its own id via put(), which becomes
|
|
111
|
+
the returned name. `name` IS still passed through as the
|
|
112
|
+
`filename=` ZeroBucket stores in its own metadata (so
|
|
113
|
+
`image.filename`/ImageMetadata.filename still reflects the
|
|
114
|
+
original upload name, even though the Django-visible `name`
|
|
115
|
+
doesn't)."""
|
|
116
|
+
if not hasattr(content, "read"):
|
|
117
|
+
content = File(content)
|
|
118
|
+
data = content.read()
|
|
119
|
+
return self._client.put(data, filename=name)
|
|
120
|
+
|
|
121
|
+
def _open(self, name: str, mode: str = "rb"):
|
|
122
|
+
image = self._client.get(name)
|
|
123
|
+
return ContentFile(image.data, name=name)
|
|
124
|
+
|
|
125
|
+
def get_available_name(self, name: str, max_length: int | None = None) -> str:
|
|
126
|
+
"""Overridden to skip Django's default collision-avoidance loop
|
|
127
|
+
entirely (which calls exists() repeatedly, appending suffixes
|
|
128
|
+
until free) -- meaningless here, since _save() above ignores
|
|
129
|
+
whatever name is passed in and always gets a fresh id from
|
|
130
|
+
put() regardless. Returning `name` unchanged avoids a wasted
|
|
131
|
+
exists() round trip for a check that could never fail to be
|
|
132
|
+
"available" in any way that matters."""
|
|
133
|
+
return name
|
|
134
|
+
|
|
135
|
+
def exists(self, name: str) -> bool:
|
|
136
|
+
return self._client.exists(name)
|
|
137
|
+
|
|
138
|
+
def delete(self, name: str) -> None:
|
|
139
|
+
self._client.delete(name)
|
|
140
|
+
|
|
141
|
+
def size(self, name: str) -> int:
|
|
142
|
+
"""Uses metadata() rather than get() -- doesn't pull the
|
|
143
|
+
(potentially large) image bytes just to answer "how big is
|
|
144
|
+
this," the same efficiency reasoning as everywhere else in
|
|
145
|
+
ZeroBucket that offers a metadata-only path."""
|
|
146
|
+
return self._client.metadata(name).size_bytes
|
|
147
|
+
|
|
148
|
+
def url(self, name: str) -> str:
|
|
149
|
+
"""Reverses to this package's own serving view (see views.py) --
|
|
150
|
+
requires `django_zerobucket.urls` to be included in your
|
|
151
|
+
project's urlconf (see that module's docstring), or this raises
|
|
152
|
+
Django's normal NoReverseMatch. Override this method in a
|
|
153
|
+
subclass if you want images served some other way instead (e.g.
|
|
154
|
+
a CDN in front of tiered S3 objects, or S3 presigned URLs
|
|
155
|
+
generated directly for images you've tiered via
|
|
156
|
+
tier_to_object_storage() -- this class has no special knowledge
|
|
157
|
+
of tiering one way or the other, since transparent reads mean
|
|
158
|
+
it doesn't need any)."""
|
|
159
|
+
return reverse("django_zerobucket:serve", args=[name])
|
|
160
|
+
|
|
161
|
+
# ---- niceties, not required by Storage but cheap to provide -------
|
|
162
|
+
|
|
163
|
+
def get_accessed_time(self, name: str):
|
|
164
|
+
raise NotImplementedError("ZeroBucket does not track last-accessed time.")
|
|
165
|
+
|
|
166
|
+
def get_created_time(self, name: str):
|
|
167
|
+
# ImageMetadata doesn't currently expose created_at -- documented
|
|
168
|
+
# gap, not silently wrong. Raising (Django's own convention for
|
|
169
|
+
# "this storage doesn't support this") rather than returning a
|
|
170
|
+
# fabricated value.
|
|
171
|
+
raise NotImplementedError(
|
|
172
|
+
"ZeroBucket's metadata() does not currently expose created_at."
|
|
173
|
+
)
|
|
174
|
+
|
|
175
|
+
def get_modified_time(self, name: str):
|
|
176
|
+
raise NotImplementedError(
|
|
177
|
+
"ZeroBucket's metadata() does not currently expose an "
|
|
178
|
+
"updated-at timestamp through the client API."
|
|
179
|
+
)
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""Include this in your project's urlconf for ZeroBucketStorage.url()
|
|
2
|
+
to resolve -- see views.py's module docstring for the access-control
|
|
3
|
+
note (no built-in auth; wrap ServeImageView yourself if you need it).
|
|
4
|
+
|
|
5
|
+
urlpatterns = [
|
|
6
|
+
...,
|
|
7
|
+
path("zerobucket-images/", include("django_zerobucket.urls")),
|
|
8
|
+
]
|
|
9
|
+
|
|
10
|
+
The path prefix ("zerobucket-images/" above) is entirely up to you --
|
|
11
|
+
ZeroBucketStorage.url() only cares about the reverse() name
|
|
12
|
+
("django_zerobucket:serve"), not the actual URL shape, so you can mount
|
|
13
|
+
this wherever fits your project.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from django.urls import path
|
|
19
|
+
|
|
20
|
+
from .views import ServeImageView
|
|
21
|
+
|
|
22
|
+
app_name = "django_zerobucket"
|
|
23
|
+
|
|
24
|
+
urlpatterns = [
|
|
25
|
+
path("<uuid:image_id>/", ServeImageView.as_view(), name="serve"),
|
|
26
|
+
]
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""The serving half of ZeroBucketStorage.url() -- a class-based view
|
|
2
|
+
that streams a stored image's bytes back over HTTP.
|
|
3
|
+
|
|
4
|
+
Include this app's urls in your project for ZeroBucketStorage.url() to
|
|
5
|
+
resolve:
|
|
6
|
+
|
|
7
|
+
# project urls.py
|
|
8
|
+
from django.urls import include, path
|
|
9
|
+
urlpatterns = [
|
|
10
|
+
...,
|
|
11
|
+
path("zerobucket-images/", include("django_zerobucket.urls")),
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
No built-in access control. This is a deliberate first-pass scope
|
|
15
|
+
decision, not an oversight: ZeroBucket's own before_get/before_put hooks
|
|
16
|
+
(see the core library's README) exist for exactly this, but wiring a
|
|
17
|
+
context= through Django's request/auth system into those hooks is a
|
|
18
|
+
real design surface of its own (whose context? the request? the user?
|
|
19
|
+
something else?) that wasn't part of what was asked for this round. If
|
|
20
|
+
you need access control on served images today, wrap this view (or a
|
|
21
|
+
subclass of it) with Django's own decorators the normal way:
|
|
22
|
+
|
|
23
|
+
from django.contrib.auth.decorators import login_required
|
|
24
|
+
from django_zerobucket.views import ServeImageView
|
|
25
|
+
|
|
26
|
+
urlpatterns = [
|
|
27
|
+
path("images/<uuid:image_id>/",
|
|
28
|
+
login_required(ServeImageView.as_view()),
|
|
29
|
+
name="my_protected_image"),
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
(and skip including django_zerobucket.urls' own unprotected route, or
|
|
33
|
+
your images will still be reachable there too.)
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
from __future__ import annotations
|
|
37
|
+
|
|
38
|
+
from django.http import (
|
|
39
|
+
HttpRequest,
|
|
40
|
+
HttpResponse,
|
|
41
|
+
HttpResponseNotFound,
|
|
42
|
+
StreamingHttpResponse,
|
|
43
|
+
)
|
|
44
|
+
from django.views import View
|
|
45
|
+
from zerobucket import ImageNotFoundError
|
|
46
|
+
|
|
47
|
+
from .storage import ZeroBucketStorage
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ServeImageView(View):
|
|
51
|
+
"""`storage_class` is instantiated ONCE, at class-definition/import
|
|
52
|
+
time (a class attribute, not per-request) -- see storage.py's
|
|
53
|
+
`_get_shared_client` for why that doesn't mean a separate connection
|
|
54
|
+
pool from whatever ZeroBucketStorage instance your FileField/
|
|
55
|
+
ImageField actually uses: both share ZeroBucket instances keyed by
|
|
56
|
+
(database_url, options), so as long as this view's implicit
|
|
57
|
+
`ZeroBucketStorage()` (using default OPTIONS -- i.e.
|
|
58
|
+
settings.ZEROBUCKET_DATABASE_URL, no extra kwargs) matches how your
|
|
59
|
+
model field's storage is configured, they resolve to the exact same
|
|
60
|
+
underlying client. If your model field's storage is configured with
|
|
61
|
+
OPTIONS beyond a bare database_url, override `storage_class` (or
|
|
62
|
+
the `storage` property below) in a subclass to match, or this view
|
|
63
|
+
will open a second, differently-configured client instead of
|
|
64
|
+
sharing the first.
|
|
65
|
+
|
|
66
|
+
Streams via get_stream(), not get() -- a real use of the core
|
|
67
|
+
library's own streaming feature, not just a demonstration of it:
|
|
68
|
+
avoids holding a full image in this process's memory during a
|
|
69
|
+
request, the exact same benefit ZeroBucket's own README describes
|
|
70
|
+
for get_stream() generally.
|
|
71
|
+
"""
|
|
72
|
+
|
|
73
|
+
storage_class = ZeroBucketStorage
|
|
74
|
+
|
|
75
|
+
@property
|
|
76
|
+
def storage(self) -> ZeroBucketStorage:
|
|
77
|
+
if not hasattr(self, "_storage"):
|
|
78
|
+
self._storage = self.storage_class()
|
|
79
|
+
return self._storage
|
|
80
|
+
|
|
81
|
+
def get(self, request: HttpRequest, image_id: str) -> HttpResponse:
|
|
82
|
+
"""Two round trips per request, stated plainly rather than left
|
|
83
|
+
for someone to notice under profiling: one explicit metadata()
|
|
84
|
+
call here (to get mime_type/size/filename for the response
|
|
85
|
+
headers before any bytes are sent), and one implicit one inside
|
|
86
|
+
get_stream() itself (its own existence/size check -- see the
|
|
87
|
+
core library's get_stream() docstring). Both are metadata-only
|
|
88
|
+
queries (no image bytes fetched twice), so this costs two small
|
|
89
|
+
round trips, not two full-image reads -- but it is two, not
|
|
90
|
+
one, and get_stream()'s public API doesn't currently offer a
|
|
91
|
+
way to skip its internal check when the caller already knows
|
|
92
|
+
the row exists."""
|
|
93
|
+
client = self.storage._client # noqa: SLF001 -- same package, deliberate reuse
|
|
94
|
+
try:
|
|
95
|
+
metadata = client.metadata(image_id)
|
|
96
|
+
except ImageNotFoundError:
|
|
97
|
+
return HttpResponseNotFound()
|
|
98
|
+
|
|
99
|
+
stream = client.get_stream(image_id)
|
|
100
|
+
response = StreamingHttpResponse(stream, content_type=metadata.mime_type)
|
|
101
|
+
response["Content-Length"] = str(metadata.size_bytes)
|
|
102
|
+
if metadata.filename:
|
|
103
|
+
response["Content-Disposition"] = f'inline; filename="{metadata.filename}"'
|
|
104
|
+
return response
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: django-zerobucket
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Django Storage backend for ZeroBucket -- database-native image storage via FileField/ImageField.
|
|
5
|
+
Project-URL: Homepage, https://github.com/KedarGhadyalji/ZeroBucket
|
|
6
|
+
Project-URL: Repository, https://github.com/KedarGhadyalji/ZeroBucket
|
|
7
|
+
Project-URL: Issues, https://github.com/KedarGhadyalji/ZeroBucket/issues
|
|
8
|
+
Project-URL: Changelog, https://github.com/KedarGhadyalji/ZeroBucket/blob/main/CHANGELOG.md
|
|
9
|
+
Author-email: Kedar Ghadyalji <your-email@example.com>
|
|
10
|
+
License-Expression: MIT
|
|
11
|
+
Keywords: django,image,postgresql,storage,zerobucket
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Framework :: Django
|
|
14
|
+
Classifier: Framework :: Django :: 5.2
|
|
15
|
+
Classifier: Framework :: Django :: 6.0
|
|
16
|
+
Classifier: Intended Audience :: Developers
|
|
17
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
18
|
+
Classifier: Operating System :: OS Independent
|
|
19
|
+
Classifier: Programming Language :: Python :: 3
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
23
|
+
Classifier: Topic :: Database
|
|
24
|
+
Classifier: Topic :: Multimedia :: Graphics
|
|
25
|
+
Classifier: Typing :: Typed
|
|
26
|
+
Requires-Python: >=3.10
|
|
27
|
+
Requires-Dist: django>=5.2
|
|
28
|
+
Requires-Dist: zerobucket>=0.15.0
|
|
29
|
+
Provides-Extra: dev
|
|
30
|
+
Requires-Dist: boto3>=1.34; extra == 'dev'
|
|
31
|
+
Requires-Dist: django-stubs>=5.0; extra == 'dev'
|
|
32
|
+
Requires-Dist: moto[s3]>=5.0; extra == 'dev'
|
|
33
|
+
Requires-Dist: mypy>=1.10; extra == 'dev'
|
|
34
|
+
Requires-Dist: pytest-cov>=5.0; extra == 'dev'
|
|
35
|
+
Requires-Dist: pytest-django>=4.8; extra == 'dev'
|
|
36
|
+
Requires-Dist: pytest>=8.0; extra == 'dev'
|
|
37
|
+
Requires-Dist: ruff>=0.6; extra == 'dev'
|
|
38
|
+
Provides-Extra: s3
|
|
39
|
+
Requires-Dist: zerobucket[s3]>=0.15.0; extra == 's3'
|
|
40
|
+
Description-Content-Type: text/markdown
|
|
41
|
+
|
|
42
|
+
# django-zerobucket
|
|
43
|
+
|
|
44
|
+
A Django [`Storage`](https://docs.djangoproject.com/en/stable/ref/files/storage/)
|
|
45
|
+
backend adapter for [ZeroBucket](https://github.com/KedarGhadyalji/ZeroBucket)
|
|
46
|
+
-- database-native image storage, plugged into Django's existing
|
|
47
|
+
`FileField`/`ImageField` machinery. No new field type to learn, works
|
|
48
|
+
with existing forms/admin/migrations the same way `django-storages`'
|
|
49
|
+
S3/GCS backends do for their targets.
|
|
50
|
+
|
|
51
|
+
## Installation
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
pip install django-zerobucket
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Requires Django >= 5.2 and a PostgreSQL database ZeroBucket can store
|
|
58
|
+
images in (see the [core `zerobucket` package](https://github.com/KedarGhadyalji/ZeroBucket)
|
|
59
|
+
for schema/setup details -- this package doesn't change any of that,
|
|
60
|
+
it's a thin adapter on top).
|
|
61
|
+
|
|
62
|
+
## Setup
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
# settings.py
|
|
66
|
+
INSTALLED_APPS = [
|
|
67
|
+
...,
|
|
68
|
+
"django_zerobucket", # needed for the zerobucket_* management commands
|
|
69
|
+
]
|
|
70
|
+
|
|
71
|
+
STORAGES = {
|
|
72
|
+
"default": {
|
|
73
|
+
"BACKEND": "django_zerobucket.storage.ZeroBucketStorage",
|
|
74
|
+
"OPTIONS": {"database_url": "postgresql://user:pass@localhost/mydb"},
|
|
75
|
+
},
|
|
76
|
+
}
|
|
77
|
+
# or, instead of OPTIONS above:
|
|
78
|
+
ZEROBUCKET_DATABASE_URL = "postgresql://user:pass@localhost/mydb"
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
# project urls.py -- required for ZeroBucketStorage.url() to resolve
|
|
83
|
+
from django.urls import include, path
|
|
84
|
+
|
|
85
|
+
urlpatterns = [
|
|
86
|
+
...,
|
|
87
|
+
path("images/", include("django_zerobucket.urls")),
|
|
88
|
+
]
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
```python
|
|
92
|
+
# models.py -- just a normal ImageField/FileField
|
|
93
|
+
class Product(models.Model):
|
|
94
|
+
photo = models.ImageField(upload_to="ignored")
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
That's it. `product.photo.save(...)`, `product.photo.read()`,
|
|
98
|
+
`product.photo.url`, `product.photo.delete()` -- all work exactly like
|
|
99
|
+
any other Django Storage backend from here.
|
|
100
|
+
|
|
101
|
+
## The one thing worth knowing before you adopt this
|
|
102
|
+
|
|
103
|
+
**The `name` Django stores for each file is a ZeroBucket image id (a
|
|
104
|
+
UUID), not a filesystem-style path.** Every other Django Storage
|
|
105
|
+
backend builds a path out of `upload_to=` plus the uploaded filename
|
|
106
|
+
(e.g. `"uploads/2024/photo.jpg"`) -- ZeroBucket is _id-addressed_
|
|
107
|
+
storage, not path-addressed, so there's no "directory" for a file to
|
|
108
|
+
live in. `upload_to=` is still required by Django's field API but is
|
|
109
|
+
effectively ignored by this backend.
|
|
110
|
+
|
|
111
|
+
This is invisible for the vast majority of normal usage
|
|
112
|
+
(`instance.photo.url`, `.read()`, `.open()`, template `{{ instance.photo.url }}`
|
|
113
|
+
all work exactly the same) -- it only matters if your code inspects the
|
|
114
|
+
raw `instance.photo.name` string and expects something path-shaped. The
|
|
115
|
+
original upload filename isn't lost, though: it's preserved in
|
|
116
|
+
ZeroBucket's own metadata and retrievable via the core `zerobucket`
|
|
117
|
+
client's `metadata()`/`get()` methods, even though Django's own `name`
|
|
118
|
+
field won't show it.
|
|
119
|
+
|
|
120
|
+
## Serving images
|
|
121
|
+
|
|
122
|
+
`ZeroBucketStorage.url()` reverses to a built-in view
|
|
123
|
+
(`ServeImageView`) that streams the image back via ZeroBucket's own
|
|
124
|
+
`get_stream()` -- a real use of the core library's streaming feature,
|
|
125
|
+
not just a demonstration of it.
|
|
126
|
+
|
|
127
|
+
**No built-in access control.** This is a deliberate first-pass scope
|
|
128
|
+
decision: ZeroBucket's own `before_get`/`before_put` hooks exist for
|
|
129
|
+
exactly this, but wiring a `context=` through Django's request/auth
|
|
130
|
+
system into those hooks is a real design question of its own (whose
|
|
131
|
+
context -- the request? the user? something else?) that's out of scope
|
|
132
|
+
for this first pass. If you need access control on served images,
|
|
133
|
+
protect the view the normal Django way:
|
|
134
|
+
|
|
135
|
+
```python
|
|
136
|
+
from django.contrib.auth.decorators import login_required
|
|
137
|
+
from django_zerobucket.views import ServeImageView
|
|
138
|
+
|
|
139
|
+
urlpatterns = [
|
|
140
|
+
path("images/<uuid:image_id>/",
|
|
141
|
+
login_required(ServeImageView.as_view()),
|
|
142
|
+
name="my_protected_image"),
|
|
143
|
+
]
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
(and don't also `include("django_zerobucket.urls")`, or the same images
|
|
147
|
+
stay reachable unprotected there too.)
|
|
148
|
+
|
|
149
|
+
Want images served some other way entirely -- a CDN in front of tiered
|
|
150
|
+
S3 objects, presigned S3 URLs for images you've moved via
|
|
151
|
+
`tier_to_object_storage()`, whatever -- subclass `ZeroBucketStorage` and
|
|
152
|
+
override `url()`. This backend has no special knowledge of tiering one
|
|
153
|
+
way or the other, since transparent reads mean it doesn't need any.
|
|
154
|
+
|
|
155
|
+
## Management commands
|
|
156
|
+
|
|
157
|
+
Thin wrappers around the core `zerobucket` CLI's already-tested
|
|
158
|
+
`cmd_info`/`cmd_verify`/`cmd_tier` -- not reimplementations. Same
|
|
159
|
+
behavior, same flags, just reading `ZEROBUCKET_DATABASE_URL` from
|
|
160
|
+
Django settings instead of requiring `--database-url`:
|
|
161
|
+
|
|
162
|
+
```bash
|
|
163
|
+
./manage.py zerobucket_info
|
|
164
|
+
./manage.py zerobucket_verify
|
|
165
|
+
./manage.py zerobucket_verify --sample 100
|
|
166
|
+
./manage.py zerobucket_tier IMAGE_ID --bucket my-bucket
|
|
167
|
+
./manage.py zerobucket_tier --all --bucket my-bucket
|
|
168
|
+
./manage.py zerobucket_tier --min-size 2000000 --older-than 90 --bucket my-bucket
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
`zerobucket_tier` needs `boto3` (`pip install "zerobucket[s3]"`) the
|
|
172
|
+
same way the standalone `zerobucket tier` CLI command does -- not
|
|
173
|
+
pulled in by installing `django-zerobucket` itself. `zerobucket_verify`/
|
|
174
|
+
`zerobucket_tier` exit non-zero on failure (checksum mismatch, tier
|
|
175
|
+
failure), same convention the standalone CLI uses, so both are usable
|
|
176
|
+
in a cron job or CI/deploy step, not just interactively.
|
|
177
|
+
|
|
178
|
+
## What this package deliberately doesn't do (yet)
|
|
179
|
+
|
|
180
|
+
- **No custom model field.** A Storage adapter was judged the higher-
|
|
181
|
+
leverage, more idiomatic choice for a first pass -- it composes with
|
|
182
|
+
Django's existing `FileField`/`ImageField`, forms, and admin, rather
|
|
183
|
+
than asking every adopting project to learn a new field type.
|
|
184
|
+
- **No async support.** The core `zerobucket` package ships
|
|
185
|
+
`AsyncZeroBucket`; this package's `ZeroBucketStorage`/`ServeImageView`
|
|
186
|
+
are both built on the sync client. Django's own async views/ORM paths
|
|
187
|
+
aren't wired up to it yet.
|
|
188
|
+
- **No signal-based cleanup.** Django does not automatically delete
|
|
189
|
+
storage files when a model instance is deleted (this is a general
|
|
190
|
+
Django behavior, not specific to this backend -- confirmed directly,
|
|
191
|
+
not assumed) -- if you want images cleaned up when their owning model
|
|
192
|
+
is deleted, add a `post_delete` signal calling `instance.photo.delete()`
|
|
193
|
+
yourself, same as you would with any other Django Storage backend.
|
|
194
|
+
- **No `dedup=True`/tiering-aware URLs.** Both are fully supported at
|
|
195
|
+
the core `zerobucket` client level (pass the relevant options through
|
|
196
|
+
`STORAGES`' `OPTIONS`) and work transparently through this adapter --
|
|
197
|
+
there's just no Django-specific UI/behavior built on top of them yet
|
|
198
|
+
(e.g. an admin action to bulk-tier selected objects).
|
|
199
|
+
|
|
200
|
+
## Development
|
|
201
|
+
|
|
202
|
+
```bash
|
|
203
|
+
cd packages/django-zerobucket
|
|
204
|
+
pip install -e ".[dev]"
|
|
205
|
+
export ZEROBUCKET_TEST_DATABASE_URL=postgresql://postgres:postgres@localhost:5432/zerobucket_test
|
|
206
|
+
pytest -q
|
|
207
|
+
ruff check src tests
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
Tests run against a real PostgreSQL instance, used for two separate
|
|
211
|
+
roles simultaneously: Django's own `DATABASES` (its internal tables plus
|
|
212
|
+
this test suite's dummy `Product` model) and `ZEROBUCKET_DATABASE_URL`
|
|
213
|
+
(where ZeroBucket actually stores image bytes) -- see
|
|
214
|
+
`tests/django_settings.py`'s module docstring for why those are kept
|
|
215
|
+
conceptually distinct even when pointed at the same physical server in
|
|
216
|
+
this test setup.
|
|
217
|
+
|
|
218
|
+
## License
|
|
219
|
+
|
|
220
|
+
MIT, same as the core `zerobucket` package.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
django_zerobucket/__init__.py,sha256=l3ExPKxAgV7hgLeiaENpPk4xEWwKtwQkZkdjDU0YXyg,955
|
|
2
|
+
django_zerobucket/apps.py,sha256=S6h0ZpIr0UxFKKWxcrpTpo-Tqt_kiOHIyHFpBTR36oc,204
|
|
3
|
+
django_zerobucket/storage.py,sha256=HB7Bf0WRDg_GxJVUsR9v8CbJGjb1ExzkWNzzVLij5zw,8371
|
|
4
|
+
django_zerobucket/urls.py,sha256=IEwk_FgccwMThvjqeJdR0_E-oe8ZOZsqiPDtN5TGs_c,808
|
|
5
|
+
django_zerobucket/views.py,sha256=8odVhhO6S_U-CrF8dHHk7oTotvBETK9_c9vw647276Y,4530
|
|
6
|
+
django_zerobucket/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
7
|
+
django_zerobucket/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
8
|
+
django_zerobucket/management/commands/zerobucket_info.py,sha256=2BvJGrX40zW1t1FRhDHkHv43vSUh-AvXMupEZS0CQTw,1027
|
|
9
|
+
django_zerobucket/management/commands/zerobucket_tier.py,sha256=FwPPidYS6lDlXn2SwLJhVKGfdY1GM0FTt1bEHmAefZE,2756
|
|
10
|
+
django_zerobucket/management/commands/zerobucket_verify.py,sha256=uFlWXPBxlqEdmU13yiy5onDJfpRVCWhY2cpNmglghuE,1223
|
|
11
|
+
django_zerobucket-0.1.0.dist-info/METADATA,sha256=u3xpXpJPvlBusUqEc2rjQfZ505_a4_4sWnbaerte3nM,8909
|
|
12
|
+
django_zerobucket-0.1.0.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
|
|
13
|
+
django_zerobucket-0.1.0.dist-info/RECORD,,
|