opposable-thumbs 0.1.0__py3-none-any.whl

opposable_thumbs/__init__.py

@@ -0,0 +1,25 @@
+"""Thumbnails and image manipulation, from Python or from URL query strings."""
+
+from importlib.metadata import version
+
+from .cache import clean_cache
+from .config import Config
+from .errors import ConfigError, ThumbnailError
+from .query import thumbnail_query
+from .result import Result, Thumbnail
+from .thumbnail import handle_request, make_thumbnail, save_thumbnail
+
+__version__ = version("opposable-thumbs")
+
+__all__ = [
+    "Config",
+    "ConfigError",
+    "Result",
+    "Thumbnail",
+    "ThumbnailError",
+    "clean_cache",
+    "handle_request",
+    "make_thumbnail",
+    "save_thumbnail",
+    "thumbnail_query",
+]

opposable_thumbs/__main__.py

@@ -0,0 +1,3 @@
+from .cli import main
+
+main()

opposable_thumbs/cache.py

@@ -0,0 +1,139 @@
+import hashlib
+import os
+import re
+import time
+import uuid
+
+# only files named like cache entries are deleted, in case a cache folder
+# shares a directory with other files: "<md5>", or "<md5>.<uuid>.tmp" left by
+# an interrupted write
+CACHE_FILENAME = re.compile(r"^[0-9a-f]{32}(\.[0-9a-f]{32}\.tmp)?$")
+
+
+def cache_key(*parts):
+    # a file name for whatever identifies a cache entry
+    key = repr(parts).encode("utf-8")
+    return hashlib.md5(key, usedforsecurity=False).hexdigest()
+
+
+def is_fresh(path, max_age):
+    # cached files older than max_age seconds are treated as missing, so they
+    # are replaced. a max_age of None keeps them forever
+    try:
+        age = time.time() - os.path.getmtime(path)
+    except OSError:
+        return False
+
+    return max_age is None or age < max_age
+
+
+def write_cache_file(path, data):
+    # write to a temp file and rename it into place, so a concurrent request
+    # never reads a partly written file. the folder is created if missing
+    os.makedirs(os.path.dirname(path), exist_ok=True)
+    temp_path = f"{path}.{uuid.uuid4().hex}.tmp"
+    try:
+        with open(temp_path, "xb") as f:
+            f.write(data)
+        os.replace(temp_path, path)
+    except Exception:
+        try:
+            os.remove(temp_path)
+        except OSError:
+            pass
+        raise
+
+
+# multipliers for the units a size can be given in, e.g. "5GB"
+SIZE_UNITS = {
+    "": 1,
+    "b": 1,
+    "k": 1000,
+    "kb": 1000,
+    "m": 1000**2,
+    "mb": 1000**2,
+    "g": 1000**3,
+    "gb": 1000**3,
+    "t": 1000**4,
+    "tb": 1000**4,
+}
+
+
+def parse_size(size):
+    """A size in bytes, from a number or a string like "500MB" or "5GB"
+    (in powers of 1000). None stays None. Raises ValueError if it isn't one.
+    """
+    if size is None:
+        return None
+    if isinstance(size, int | float) and not isinstance(size, bool):
+        value = size
+    else:
+        match = re.fullmatch(r"\s*(\d+(?:\.\d+)?)\s*([a-zA-Z]*)\s*", str(size))
+        if not match or match.group(2).lower() not in SIZE_UNITS:
+            raise ValueError(f"Invalid size: {size!r}, e.g. use 500MB or 5GB")
+        value = float(match.group(1)) * SIZE_UNITS[match.group(2).lower()]
+    if value < 0:
+        raise ValueError(f"Invalid size: {size!r}, it can't be negative")
+    return int(value)
+
+
+def clean_cache(config, all=False, max_size=None):
+    """Delete files from both cache folders, and return how many.
+
+    Expired files are deleted, or every file with all=True. Then, while a
+    cache folder is bigger than max_size bytes (by default
+    config.cache_max_size), its oldest files are deleted. max_size can be a
+    number of bytes or a string like "5GB". Run it on a schedule, as files
+    that are never requested again otherwise stay on disk.
+    """
+    max_size = config.cache_max_size if max_size is None else parse_size(max_size)
+    return sum(
+        _clean_folder(folder, config.cache_max_age, all, max_size)
+        for folder in (config.input_cache_dir, config.output_cache_dir)
+        if folder and os.path.isdir(folder)
+    )
+
+
+def _clean_folder(folder, max_age, all, max_size):
+    now = time.time()
+    deleted = 0
+    kept = []
+    for entry in os.scandir(folder):
+        if not entry.is_file(follow_symlinks=False):
+            continue
+        if not CACHE_FILENAME.match(entry.name):
+            continue
+        try:
+            stat = entry.stat()
+        except FileNotFoundError:
+            continue
+
+        if all or (max_age is not None and now - stat.st_mtime >= max_age):
+            deleted += _remove(entry.path)
+        else:
+            kept.append((stat.st_mtime, stat.st_size, entry.name, entry.path))
+
+    if max_size is not None:
+        total = sum(size for _, size, _, _ in kept)
+        # oldest first. files still being written (.tmp) count towards the
+        # size, but are left alone
+        for _, size, name, path in sorted(kept):
+            if total <= max_size:
+                break
+            if name.endswith(".tmp"):
+                continue
+            if _remove(path):
+                deleted += 1
+                total -= size
+
+    return deleted
+
+
+def _remove(path):
+    # 1 if the file was deleted, 0 if it was already gone (e.g. deleted by
+    # another process)
+    try:
+        os.remove(path)
+        return 1
+    except FileNotFoundError:
+        return 0

opposable_thumbs/cli.py

@@ -0,0 +1,50 @@
+import argparse
+
+from .cache import clean_cache
+from .config import Config
+from .errors import ConfigError
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(prog="opposable-thumbs")
+    commands = parser.add_subparsers(dest="command", required=True)
+
+    clean = commands.add_parser(
+        "clean-cache",
+        help="delete expired files from the cache folders",
+        description=(
+            "Delete cached files older than --max-age seconds, then the oldest "
+            "files in any folder bigger than --max-size. Run it on a schedule "
+            "to stop the caches growing."
+        ),
+    )
+    clean.add_argument("--input-cache-dir", help="the input_cache_dir folder")
+    clean.add_argument("--output-cache-dir", help="the output_cache_dir folder")
+    clean.add_argument(
+        "--max-age",
+        type=float,
+        default=Config.cache_max_age,
+        help="the cache_max_age setting, in seconds (default: %(default)s)",
+    )
+    clean.add_argument(
+        "--max-size",
+        help="the cache_max_size setting: the size to keep each folder under, "
+        "e.g. 5GB (default: no limit)",
+    )
+    clean.add_argument("--all", action="store_true", help="delete every cached file")
+
+    args = parser.parse_args(argv)
+    if not args.input_cache_dir and not args.output_cache_dir:
+        parser.error("give --input-cache-dir, --output-cache-dir or both")
+
+    try:
+        config = Config(
+            input_cache_dir=args.input_cache_dir,
+            output_cache_dir=args.output_cache_dir,
+            cache_max_age=args.max_age,
+            cache_max_size=args.max_size,
+        )
+    except ConfigError as e:
+        parser.error(str(e))
+    deleted = clean_cache(config, all=args.all)
+    print(f"Deleted {deleted} cached file(s)")

opposable_thumbs/config.py

@@ -0,0 +1,104 @@
+import os
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, field
+from types import MappingProxyType
+
+from PIL import Image
+
+from .cache import parse_size
+from .errors import ConfigError
+
+StrPath = str | os.PathLike
+
+
+@dataclass(frozen=True)
+class Config:
+    """Settings for making thumbnails. Every setting has a safe default.
+
+    Nothing can be opened until `roots` or `allowed_urls` lists a source, and
+    URLs are signed, so serving them needs a `secret_key`.
+    """
+
+    # sources. file= is "<root name>/<path inside that root's folder>", e.g.
+    # roots={"media": "/srv/media"} and file=media/photos/a.jpg. a url is only
+    # fetched if it matches allowed_urls
+    roots: Mapping[str, StrPath] = field(default_factory=dict)
+    allowed_urls: Sequence[str] = ()
+
+    # thumbnail_query() signs URLs with secret_key, and handle_request()
+    # rejects any that aren't signed with it, so nobody can make up new ones.
+    # without signed_urls, any URL is accepted. not shown in repr(), so it
+    # doesn't end up in logs
+    secret_key: str | bytes | None = field(default=None, repr=False)
+    signed_urls: bool = True
+    # source images in any other format are rejected, to limit the Pillow
+    # parsers that untrusted input can reach
+    allowed_formats: Sequence[str] = ("JPEG", "PNG", "GIF", "WEBP", "BMP", "TIFF")
+
+    # caches, each turned on by setting its folder
+    input_cache_dir: StrPath | None = None
+    output_cache_dir: StrPath | None = None
+    # seconds before a cached file is replaced. None keeps them forever
+    cache_max_age: float | None = 7 * 24 * 60 * 60
+    # the size clean_cache() keeps each cache folder under, in bytes or as a
+    # string like "5GB". None has no limit
+    cache_max_size: int | str | None = None
+
+    # sent with every image. None sends no Cache-Control header
+    cache_control: str | None = "max-age=86400"
+    # rotate images upright using their EXIF orientation, as browsers do
+    apply_exif_orientation: bool = True
+
+    # limits on a single request
+    max_download_bytes: int = 20 * 1024 * 1024
+    fetch_timeout: float = 10
+    max_source_pixels: int = 50_000_000
+    max_output_size: int = 4000
+    max_operations: int = 10
+    max_blur_radius: int = 100
+
+    def __post_init__(self):
+        # store lists as tuples and paths as strings, so a Config can't change
+        # after it is made and can be shared between threads
+        set_ = object.__setattr__
+        set_(self, "roots", self._roots(self.roots))
+        set_(self, "allowed_urls", self._tuple("allowed_urls", self.allowed_urls))
+        if self.secret_key is not None and (
+            not isinstance(self.secret_key, str | bytes) or not self.secret_key
+        ):
+            raise ConfigError("secret_key must be a non-empty string")
+        set_(self, "allowed_formats", self._formats(self.allowed_formats))
+        try:
+            set_(self, "cache_max_size", parse_size(self.cache_max_size))
+        except ValueError as e:
+            raise ConfigError(f"cache_max_size: {e}") from e
+        for name in ("input_cache_dir", "output_cache_dir"):
+            value = getattr(self, name)
+            set_(self, name, os.fspath(value) if value else None)
+
+    @staticmethod
+    def _roots(roots):
+        if not isinstance(roots, Mapping):
+            raise ConfigError("roots must map names to folders")
+        for name in roots:
+            # the name is the first part of file=, up to the first /
+            if not isinstance(name, str) or not name or "/" in name:
+                raise ConfigError(f"Invalid roots name: {name!r}")
+        return MappingProxyType({n: os.fspath(f) for n, f in roots.items()})
+
+    @staticmethod
+    def _tuple(name, value):
+        # a single string would otherwise become a tuple of its characters
+        if isinstance(value, str | bytes | os.PathLike):
+            raise ConfigError(f"{name} must be a list, not a single value")
+        return tuple(value)
+
+    @classmethod
+    def _formats(cls, formats):
+        # an unknown name would make Image.open raise, so fail here instead
+        formats = tuple(f.upper() for f in cls._tuple("allowed_formats", formats))
+        Image.init()
+        unknown = [f for f in formats if f not in Image.OPEN]
+        if unknown:
+            raise ConfigError(f"Unknown allowed_formats: {', '.join(unknown)}")
+        return formats

opposable_thumbs/django/__init__.py

@@ -0,0 +1,13 @@
+"""Django integration: pip install opposable-thumbs[django].
+
+Add "opposable_thumbs.django" to INSTALLED_APPS, set OPPOSABLE_THUMBS in your
+settings and include "opposable_thumbs.django.urls" in your URLconf. See the
+README for the details.
+"""
+
+try:
+    import django  # noqa: F401
+except ImportError as e:
+    raise ImportError(
+        "opposable_thumbs.django needs Django: pip install opposable-thumbs[django]"
+    ) from e

opposable_thumbs/django/apps.py

@@ -0,0 +1,8 @@
+from django.apps import AppConfig
+
+
+class OpposableThumbsConfig(AppConfig):
+    name = "opposable_thumbs.django"
+    # the default label would be "django"
+    label = "opposable_thumbs"
+    verbose_name = "opposable-thumbs"

opposable_thumbs/django/conf.py

@@ -0,0 +1,34 @@
+from functools import cache
+
+from django.conf import settings
+from django.core.exceptions import ImproperlyConfigured
+from django.core.signals import setting_changed
+from django.dispatch import receiver
+from django.utils.crypto import salted_hmac
+
+from opposable_thumbs import Config, ConfigError
+
+
+@cache
+def get_config():
+    """The Config from settings.OPPOSABLE_THUMBS, a dict of Config settings.
+
+    Unless it sets secret_key, URLs are signed with a key made from
+    settings.SECRET_KEY (not SECRET_KEY itself), so they need no setting.
+    """
+    options = dict(getattr(settings, "OPPOSABLE_THUMBS", {}))
+    options.setdefault(
+        "secret_key",
+        salted_hmac("opposable_thumbs", "signing", algorithm="sha256").hexdigest(),
+    )
+    try:
+        return Config(**options)
+    except (TypeError, ConfigError) as e:
+        raise ImproperlyConfigured(f"OPPOSABLE_THUMBS: {e}") from e
+
+
+@receiver(setting_changed)
+def _settings_changed(setting, **kwargs):
+    # e.g. override_settings in tests
+    if setting in ("OPPOSABLE_THUMBS", "SECRET_KEY"):
+        get_config.cache_clear()

opposable_thumbs/django/management/commands/clean_opposable_cache.py

@@ -0,0 +1,32 @@
+from django.core.management.base import BaseCommand, CommandError
+
+from opposable_thumbs import clean_cache
+
+from ...conf import get_config
+
+
+class Command(BaseCommand):
+    help = (
+        "Delete cached files older than cache_max_age from the cache folders "
+        "in OPPOSABLE_THUMBS, then the oldest files in any folder bigger than "
+        "cache_max_size. Run it on a schedule to stop the caches growing."
+    )
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--all", action="store_true", help="Delete every cached file."
+        )
+        parser.add_argument(
+            "--max-size",
+            help="The size to keep each folder under, e.g. 5GB, instead of "
+            "cache_max_size.",
+        )
+
+    def handle(self, *args, **options):
+        try:
+            deleted = clean_cache(
+                get_config(), all=options["all"], max_size=options["max_size"]
+            )
+        except ValueError as e:
+            raise CommandError(e) from e
+        self.stdout.write(f"Deleted {deleted} cached file(s)")

opposable_thumbs/django/templatetags/opposable_thumbs.py

@@ -0,0 +1,16 @@
+from django import template
+
+from ..views import thumbnail_url as _thumbnail_url
+
+register = template.Library()
+
+
+@register.simple_tag
+def thumbnail_url(file=None, p="", *, url=None, format=None, quality=None):
+    """The signed URL of a thumbnail, e.g.
+
+    {% thumbnail_url "media/a.jpg" "fit,300,300|grayscale" format="webp" %}
+
+    The operations are written as in the p= query parameter.
+    """
+    return _thumbnail_url(file, url=url, operations=p, format=format, quality=quality)

opposable_thumbs/django/urls.py

@@ -0,0 +1,8 @@
+from django.urls import path
+
+from . import views
+
+# include() this at the path thumbnails should be served from, e.g.
+# path("thumb/", include("opposable_thumbs.django.urls"))
+app_name = "opposable_thumbs"
+urlpatterns = [path("", views.thumbnail, name="thumbnail")]

opposable_thumbs/django/views.py

@@ -0,0 +1,40 @@
+from django.http import HttpResponse
+from django.urls import reverse
+from django.views.decorators.http import require_safe
+
+from opposable_thumbs import handle_request, thumbnail_query
+
+from .conf import get_config
+
+
+@require_safe
+def thumbnail(request):
+    result = handle_request(
+        get_config(),
+        request.META.get("QUERY_STRING", ""),
+        if_none_match=request.headers.get("If-None-Match"),
+    )
+    response = HttpResponse(result.body, status=result.status)
+    for name, value in result.headers.items():
+        response[name] = value
+    if "Content-Type" not in result.headers:
+        # a 304 has none, and Django would otherwise add text/html
+        del response["Content-Type"]
+    return response
+
+
+def thumbnail_url(file=None, *, url=None, operations=(), format=None, quality=None):
+    """The URL of a thumbnail, signed, for use in views and templates.
+
+    Takes the same arguments as make_thumbnail(), e.g.
+    thumbnail_url("media/a.jpg", operations=[("fit", 300, 300)]).
+    """
+    query = thumbnail_query(
+        get_config(),
+        file=file,
+        url=url,
+        operations=operations,
+        format=format,
+        quality=quality,
+    )
+    return f"{reverse('opposable_thumbs:thumbnail')}?{query}"

opposable_thumbs/errors.py

@@ -0,0 +1,15 @@
+class ConfigError(ValueError):
+    """A Config setting is invalid."""
+
+
+class ThumbnailError(Exception):
+    """save_thumbnail() couldn't make the thumbnail.
+
+    `errors` lists every problem found, and `status` is the HTTP status code
+    make_thumbnail() would have returned for them.
+    """
+
+    def __init__(self, errors, status):
+        super().__init__("\n".join(errors))
+        self.errors = tuple(errors)
+        self.status = status

opposable_thumbs/flask.py

@@ -0,0 +1,93 @@
+"""Flask integration: pip install opposable-thumbs[flask].
+
+    from opposable_thumbs.flask import init_app
+
+    app.config["OPPOSABLE_THUMBS"] = {"roots": {"media": "/srv/media"}}
+    init_app(app)
+
+See the README for the details.
+"""
+
+import hashlib
+import hmac
+
+try:
+    import click
+    from flask import Response, current_app, request, url_for
+except ImportError as e:
+    raise ImportError(
+        "opposable_thumbs.flask needs Flask: pip install opposable-thumbs[flask]"
+    ) from e
+
+from opposable_thumbs import Config, clean_cache, handle_request, thumbnail_query
+
+EXTENSION = "opposable_thumbs"
+
+
+def init_app(app, url="/thumb", config=None):
+    """Serve thumbnails at `url`, and add thumbnail_url() to templates.
+
+    The Config is `config` if given, otherwise made from
+    app.config["OPPOSABLE_THUMBS"], a dict of Config settings. Unless that
+    sets secret_key, URLs are signed with a key made from app.secret_key (not
+    app.secret_key itself).
+    """
+    if config is None:
+        options = dict(app.config.get("OPPOSABLE_THUMBS", {}))
+        if "secret_key" not in options and app.secret_key:
+            secret = app.secret_key
+            if isinstance(secret, str):
+                secret = secret.encode("utf-8")
+            options["secret_key"] = hmac.new(
+                secret, b"opposable_thumbs.signing", hashlib.sha256
+            ).hexdigest()
+        config = Config(**options)
+
+    app.extensions[EXTENSION] = config
+    app.add_url_rule(url, endpoint=EXTENSION, view_func=_thumbnail)
+    app.add_template_global(thumbnail_url)
+
+    @app.cli.command("clean-thumbnail-cache")
+    @click.option("--all", "all_", is_flag=True, help="Delete every cached file.")
+    @click.option(
+        "--max-size",
+        help="The size to keep each folder under, e.g. 5GB, instead of cache_max_size.",
+    )
+    def clean_thumbnail_cache(all_, max_size):
+        """Delete expired files from the thumbnail cache folders, then the
+        oldest files in any folder bigger than cache_max_size."""
+        try:
+            deleted = clean_cache(config, all=all_, max_size=max_size)
+        except ValueError as e:
+            raise click.BadParameter(str(e), param_hint="--max-size") from e
+        click.echo(f"Deleted {deleted} cached file(s)")
+
+
+def _thumbnail():
+    result = handle_request(
+        current_app.extensions[EXTENSION],
+        request.query_string.decode("utf-8", errors="replace"),
+        if_none_match=request.headers.get("If-None-Match"),
+    )
+    response = Response(result.body, status=result.status, headers=result.headers)
+    if "Content-Type" not in result.headers:
+        # a 304 has none, and Flask would otherwise add text/html
+        del response.headers["Content-Type"]
+    return response
+
+
+def thumbnail_url(file=None, *, url=None, operations=(), format=None, quality=None):
+    """The URL of a thumbnail, signed, for use in views and templates.
+
+    Takes the same arguments as make_thumbnail(), e.g. in a template
+    {{ thumbnail_url("media/a.jpg", operations="fit,300,300|grayscale") }}.
+    """
+    query = thumbnail_query(
+        current_app.extensions[EXTENSION],
+        file=file,
+        url=url,
+        operations=operations,
+        format=format,
+        quality=quality,
+    )
+    return f"{url_for(EXTENSION)}?{query}"