oxylus-utils 0.0.1__py2.py3-none-any.whl

ox_utils/django/__init__.py

@@ -0,0 +1,3 @@
+"""
+This module provide utilities and tools to work with Django.
+"""

ox_utils/django/conf.py

@@ -0,0 +1,111 @@
+from functools import cached_property
+from pathlib import Path
+from typing import Any
+
+import yaml
+from django.conf import settings
+
+
+class Settings:
+    """
+    This class provide settings configuration with default values.
+
+    Configuration keys are declared and document under subclass of this
+    one as UPPERCASED attributes. This allows to document them.
+
+    User can still import the class in ``settings.py`` in order to
+    reuse already existing values.
+
+
+    Example ``myapp.conf``
+
+    .. code-block:: python
+
+        from ox_utils.django import conf
+
+        __all__ = ("MyAppSettings", "app_settings")
+
+        class MyAppSettings(conf.Settings):
+            ALLOWED_TAGS = ["a", "b", "br"]
+            \""" Tags allowed for this application. \"""
+            SOME_VALUE = 123
+            \""" Other documented setting. \"""
+
+        # this is the actual instance used to retrieve settings
+        app_settings = MyAppSettings("MYAPP")
+
+    In project ``settings.py``:
+
+    .. code-block:: python
+
+        from myapp.settings import MyAppSettings
+
+        MYAPP = {
+            "ALLOWED_TAGS": [*MyAppSettings.ALLOWED_TAGS, "p"],
+        }
+
+
+    Usage:
+
+    .. code-block:: python
+
+        from myapp.conf import app_settings
+
+        # Just print it out...
+        print(app_settings.ALLOWED_TAGS, app_settings.SOME_VALUE)
+
+    """
+
+    def __init__(self, key, source=settings):
+        self._overrides = getattr(source, key, None)
+
+    def __getattribute__(self, name):
+        if name[0] != "_" and name.isupper():
+            overrides = object.__getattribute__(self, "_overrides")
+            if overrides and name in overrides:
+                return overrides[name]
+        return object.__getattribute__(self, name)
+
+
+class SettingsEditor:
+    """
+    Helper class to work with dynaconf settings files, one per setting file.
+    """
+
+    name: str
+    """ Settings name """
+    dir: Path
+    """ Path to settings directory. """
+
+    def __init__(self, name, dir):
+        self.name = name
+        self.dir = Path(dir)
+
+    @classmethod
+    def from_app(cls, app, *args, **kwargs):
+        """Return SettingsEditor for the provided file."""
+        name = f"{app.label}.yaml"
+        return cls(name, *args, **kwargs)
+
+    @cached_property
+    def path(self):
+        """ """
+        return self.dir / self.name
+
+    def _ensure(self):
+        """Ensure directory exists."""
+        self.dir.mkdir(parents=True, exist_ok=True)
+
+    def read(self) -> dict[str, Any] | None:
+        """Read data from settings."""
+        if not self.path.exists():
+            return None
+
+        with open(self.path, "r") as stream:
+            return yaml.load(stream, Loader=yaml.Loader)
+
+    def write(self, data):
+        """Write data to settings file."""
+        self._ensure()
+        with open(self.path, "w") as stream:
+            yaml.dump(data, stream)

ox_utils/django/files/__init__.py

@@ -0,0 +1,21 @@
+"""
+This module provides files utilities for Django.
+
+It provides:
+
+    - Files mime-type detection based on file content.
+    - Reusable File and Folder abstract models and configuration.
+
+Required dependencies (with dependency group: ``files``):
+
+- PIL: used for ImageProcessor & PDFProcessor
+- libreoffice: LibreOfficeProcessor
+- pymupdf: used PDFProcessor
+
+"""
+from . import processors, models
+
+__all__ = (
+    "processors", "models"
+)
+

ox_utils/django/files/conf.py

@@ -0,0 +1,71 @@
+from pathlib import Path
+from uuid import UUID
+
+from django.conf import settings
+from .. import conf
+
+
+__all__ = ("FileSettings", "files_settings", "resolve")
+
+
+class FileSettings(conf.Settings):
+    """
+    Setting class used by the files module.
+    """
+    MEDIA_DIR: Path = settings.MEDIA_ROOT / "ox_files"
+    """ MEDIA directory """
+    UPLOAD_DIR: str = "ox_files/uploads"
+    """ Subdirectory in media dir where to upload files. """
+    PREVIEW_DIR: str = "ox_files/previews"
+    """ Directory in media dir which to save previews. """
+
+    FILE_SIZE_LIMIT = 15 * 1024 * 1024
+    """ Set maximum file size. """
+
+    MAGIC_BUFFER: int = 2048
+    """ Buffer size used by Python-Magic to read mime types. """
+    THUMBNAIL_SIZE: tuple[int, int] = (600, 800)
+    """ Size for thumbnails """
+
+    PROCESSORS: list[str] = [
+        "ox_utils.django.files.processors.ImageProcessor",
+        "ox_utils.django.files.processors.PDFProcessor",
+        "ox_utils.django.files.processors.LibreOfficeProcessor",
+    ]
+    """ List of file processors. """
+
+    @property
+    def upload_dir(self) -> Path:
+        """Return full path to upload dir."""
+        # We enforce media to be protected
+        return self.MEDIA_DIR / self.UPLOAD_DIR
+
+    @property
+    def preview_dir(self) -> Path:
+        """Return full path to preview dir."""
+        # We enforce media to be protected
+        return self.MEDIA_DIR / self.PREVIEW_DIR
+
+    def resolve(self, path: str, relative: bool = False):
+        """Resolve path under file upload directory.
+
+        :param path: path to resolve.
+        :param relative: if True, return path relative to MEDIA_ROOT.
+        """
+        if path.startswith("/"):
+            path = path[1:]
+
+        path = self.upload_dir / path
+        if relative:
+            return path.relative_to(settings.MEDIA_ROOT)
+        return path
+
+
+files_settings: FileSettings = FileSettings("OX_FILES")
+"""
+Settings used by file models by default, using key ``OX_FILES``.
+"""
+
+resolve = files_settings.resolve
+""" Shortcut to ``files_settings`` 's :py:meth:`Settings.resolve`. """
+

ox_utils/django/files/exceptions.py

@@ -0,0 +1,7 @@
+__all__ = ("FileTypeError",)
+
+
+class FileTypeError(ValueError):
+    """Invalid file format."""
+
+    pass

ox_utils/django/files/models/__init__.py

@@ -0,0 +1,4 @@
+from .folder import BaseFolder, FolderQuerySet
+from .file import BaseFile, FileQuerySet
+
+__all__ = ("BaseFolder", "FolderQuerySet", "BaseFile", "FileQuerySet")

ox_utils/django/files/models/file.py

@@ -0,0 +1,262 @@
+from __future__ import annotations
+
+from functools import cached_property
+from pathlib import Path
+import os
+import shutil
+from typing import Type
+
+from django.db import models, transaction
+from django.conf import settings
+from django.core.exceptions import ValidationError, PermissionDenied, FieldDoesNotExist
+from django.utils.translation import gettext_lazy as _
+from django.urls import reverse
+
+from ox_utils.django.models import Described, Timestamped, UUID
+
+from ..conf import files_settings, FileSettings
+from .. import processors
+from .folder import BaseFolder, validate_name
+
+
+__all__ = ("FileQuerySet", "BaseFile", "file_upload_to")
+
+
+class FileQuerySet(models.QuerySet):
+    def delete(self, clear_files: bool | None = None):
+        if clear_files is None:
+            clear_files = self.model._settings.CLEAR_FILES_ON_DELETE
+
+        if clear_files:
+            self.clear_files(update=False)
+        return super().delete()
+
+    def clear_files(self, update: bool = True):
+        """
+        Delete files from filesystem and optionally update fields.
+
+        :param update: if True (default), update queryset fields.
+        """
+        for file, preview in self.values_list("file", "preview"):
+            (settings.MEDIA_ROOT / file).unlink(missing_ok=True)
+            preview and (settings.MEDIA_ROOT / preview).unlink(missing_ok=True)
+
+        if update:
+            self.update(file=None, preview=None)
+
+
+def file_upload_to(instance, filename) -> str:
+    """Return target upload file depending on whether the folder is synchronized or not."""
+    if instance.folder_model() and instance.folder:
+        path = f"{instance.folder.path}/{filename}"
+    else:
+        path = f"/{filename}"
+    return instance._settings.resolve(path, relative=True)
+
+
+class BaseFile(Described, Timestamped, UUID):
+    """
+    This class represent a file.
+
+    Files are looked up for a matching :py:class:`~.processors.FileProcessor` (reading mime-type). A processors
+    handles different tasks such as creating preview (thumbnails) or getting metadata. In order to do so, it uses
+    :py:data:`~.processors.registry` that provides helpers to read mime types and register file processors.
+
+    Folders
+    -------
+
+    A file can be nested under a :py:attr:`parent` folder. If ``None`` is provided, then it will be
+    at the root of the filesystem.
+
+    If you want to add folder, you must add the ``folder`` foreign key:
+
+        - Reverse name as ``files``.
+        - One Folder concrete subclass per File subclass.
+
+    Settings
+    --------
+
+    The model uses by default the :py:attr:`~ox_utils.django.files.conf.files_settings`. However
+    you might want to integrate your own FileSettings instance by assigning :py:attr:`_settings`.
+
+    At deletion, related files and previews are deleted based on ``.conf.files_settings`` (``CLEAR_FILES_ON_DELETE=True`` option).
+    """
+
+    _settings = files_settings
+    """ Specify settings to use for files. """
+
+    # When folder is null, it is at root
+    # folder = models.ForeignKey(Folder, models.CASCADE, related_name="files", verbose_name=_("Folder"))
+
+    name = models.CharField(_("Name"), max_length=128, validators=[validate_name])
+    file = models.FileField(_("File"), upload_to=file_upload_to, null=True)
+    preview = models.FileField(_("Preview"), null=True, blank=True)
+    mime_type = models.CharField(_("Mime Type"), max_length=127, blank=True)
+    file_size = models.PositiveIntegerField(_("File Size"), blank=True, default=0)
+    preview_size = models.PositiveIntegerField(_("Preview Size"), blank=True, default=0)
+
+    caption = models.TextField(
+        _("Caption"),
+        default="",
+        help_text=_("Displayed below object when rendered in page or documents."),
+        blank=True,
+        null=True,
+    )
+    alternate = models.TextField(
+        _("Alternate text"),
+        default="",
+        help_text=_("Displayed as replacement text when object is not displayed."),
+        blank=True,
+        null=True,
+    )
+    ariaDescription = models.TextField(
+        _("ARIA Description"),
+        default="",
+        blank=True,
+        null=True,
+        help_text=_("Provide a description of the content for visually impaired people."),
+    )
+    metadata = models.JSONField(_("Metadata"), default=dict, blank=True)
+
+    objects = FileQuerySet.as_manager()
+
+    class Meta:
+        abstract = True
+        verbose_name = _("File")
+        verbose_name_plural = _("Files")
+        # FIXME: add unique constraint
+
+    @cached_property
+    def file_url(self):
+        """Return file url."""
+        uuid = self.access and self.access.uuid or self.uuid
+        return reverse("serve-file", kwargs={"uuid": uuid})
+
+    @cached_property
+    def preview_url(self):
+        """Return file preview url."""
+        if not self.preview:
+            return None
+        uuid = self.access and self.access.uuid or self.uuid
+        return reverse("serve-file-preview", kwargs={"uuid": uuid})
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self._initial_path = self.file and self.file.path
+
+    @classmethod
+    def folder_model(cls) -> Type[BaseFolder]:
+        """
+        Return folder class used by this model.
+
+        It assumes that either it is a foreignkey to "folder" or not declared.
+        """
+        try:
+            return cls.get_field("folder").target.remote_field.model
+        except FieldDoesNotExist:
+            return None
+
+    def abs_path(self):
+        return self.folder.abs_path() / self.name
+
+    def on_save(self, fields=None):
+        """Ensure mime type and file validation."""
+        super().on_save(fields)
+        self.validate_node()
+
+    def validate_node(self):
+        """
+        Ensure file name is unique within folder.
+
+        :yield ValidationError: if file name is already present in folder (file or folder).
+        """
+        if Folder := self.folder_model():
+            # TODO: handle file move
+            kw = self._get_validation_lookup()
+
+            query = type(self).objects.filter(folder_id=self.folder_id, **kw)
+            if self.pk:
+                query = query.exclude(pk=self.pk)
+
+            if query.exists():
+                raise ValidationError({"name": "Another file exists for this path."})
+
+            if Folder.objects.filter(parent_id=self.folder_id, **kw):
+                raise ValidationError({"name": "A folder exists for this path."})
+
+
+    def _get_validation_lookup(self):
+        return {"name": self.name}
+
+    def read_mime_type(self, save: bool = True) -> str:
+        """Read mime-type from file and update corresponding field.
+
+        :param save: save object instance
+        :return the mime type
+        """
+        self.file_size = Path(self.file.path).stat().st_size
+        self.mime_type = processors.registry.read_mime_type(self.file.path)
+        save and self.save()
+        return self.mime_type
+
+    def get_processor(
+        self, save: bool = True, registry: processors.FileProcessors = processors.registry
+    ) -> processors.Processor:
+        """Return processor to use for this file type.
+
+        Read mime-type from file if not already set.
+
+        :param save: save model if mime type is updated (default is True)
+        :param registry: use this processors instead of :py:data:`.processors.processors`
+        """
+        if not self.mime_type:
+            self.read_mime_type(save)
+        return registry.get(self.mime_type)
+
+    def create_preview(self, force: bool = True) -> tuple[bool,Path]:
+        """
+        Generate and assign a preview to this file instance.
+
+        :param force: if True (default), create preview even if already present.
+        :return a tuple of ``created, preview_path``.
+        """
+        if self.preview:
+            if not force:
+                return (False, self.preview.path)
+            self.preview.delete()
+            self.preview = None
+            self.save(update_fields=["preview"])
+
+        with transaction.atomic():
+            size = self._settings.THUMBNAIL_SIZE
+            path = self._settings.preview_dir / f"{self.uuid}-{size}.jpg"
+
+            path.parent.mkdir(parents=True, exist_ok=True)
+            created = processor.create_preview(Path(self.file.path), path, size, force=True)
+
+            if created:
+                self.preview.name = str(path.relative_to(settings.MEDIA_ROOT))
+                self.save()
+        return (created, path)
+
+    def delete(self, *args, clear_files: bool | None = None, **kwargs):
+        """
+        Ensure file deletion.
+
+        :param *args: forward to super's ``delete()``
+        :param clear_files: if True or False, overrides default settings.
+        :param **kwargs: forward to super's ``delete()``
+        """
+        if clear_files is None:
+            clear_files = self._settings.CLEAR_FILES_ON_DELETE
+
+        if clear_files:
+            if self.preview:
+                self.preview.delete(False)
+            if self.file:
+                self.file.delete(False)
+
+            self.preview = None
+            self.file = None
+        return super().delete(*args, **kwargs)
+

ox_utils/django/files/models/folder.py

@@ -0,0 +1,116 @@
+from __future__ import annotations
+
+from pathlib import Path
+import shutil
+
+from django.db import models
+from django.core.exceptions import ValidationError
+from django.utils.translation import gettext_lazy as _
+
+from ox_utils.django.models import Described, Timestamped
+from ox_utils.django.models.tree import TreeNode, TreeNodeQuerySet
+from ..conf import FileSettings, files_settings
+
+
+__all__ = (
+    "FolderQuerySet",
+    "BaseFolder",
+    "validate_name",
+)
+
+
+def validate_name(value):
+    """Validate folder or file name."""
+    if "/" in value:
+        raise ValidationError(_("The character `/` is forbidden in name."))
+
+
+class FolderQuerySet(TreeNodeQuerySet):
+    pass
+
+
+class BaseFolder(Described, Timestamped, TreeNode):
+    """
+    Represent a folder in which files are stored.
+
+    .. important::
+
+        Updating :py:attr:`parent`, :py:attr:`name` and :py:attr:`path` should not be done manually. Instead use
+        :py:meth:`rename` and :py:meth:`move_to` methods to ensure that these values are correctly set.
+
+        When thoses values raise a ValidationError, user should assume that new values of the model are invalid.
+
+    """
+
+    name = models.CharField(_("Name"), max_length=64, validators=[validate_name])
+
+    objects = FolderQuerySet.as_manager()
+
+    class Meta:
+        abstract = True
+        verbose_name = _("Folder")
+        verbose_name_plural = _("Folders")
+        constraints = [models.UniqueConstraint("parent", "name", name="unique_folder_name")]
+
+    @classmethod
+    def file_model(cls):
+        """ Return the file model related to this folder (assuming the reverse name is ``files``). """
+        return cls.get_field("files").related_model
+
+    def abs_path(self) -> Path:
+        """File system path of the folder."""
+        return self.file_model()._settings.resolve(self.path)
+
+    def rename(self, name: str, save: bool = True):
+        """Rename folder."""
+        if name != self.name:
+            self.name = name
+            if save:
+                self.save()
+            else:
+                self.on_save()
+
+    def move_to(self, parent: BaseFolder | None = None, name: str | None = None, save: bool = True):
+        """Move folder into provided parent folder or root.
+
+        :param parent: parent folder
+        :param name: if provided rename folder
+        :param save: save node
+        """
+        if (name and name != self.name) or parent.id != self.parent_id:
+            if name:
+                self.name = name
+            super().move_to(parent, save)
+
+    def validate_node(self):
+        """
+        Validate node for name collision (folder & file).
+
+        :yield ValidationError: a file or folder already exists with this name in parent.
+        """
+        super().validate_node()
+
+        if self.file_model().objects.filter(folder=self.parent, name=self.name):
+            raise ValidationError({"name": f"A file `{self.name}` already exists in {self.parent.name}."})
+
+    def sync_node(self, source_path: str | None = None, target_path: str | None = None):
+        """Synchronize folder with filesystem (move or create it)."""
+        source_p = source_path and self.file_model()._settings.resolve(source_path)
+        target_p = self.file_model()._settings.resolve(target_path)
+
+        if target_p.exists():
+            raise ValueError(f"A file or directory already exists at `{target_p}`.")
+
+        if source_p and source_p.exists():
+            shutil.move(source_p, target_p)
+        else:
+            target_p.mkdir(parents=True, exist_ok=True)
+
+        super().sync_node(source_path, target_path)
+
+    def delete(self):
+        path = self.file_model()._settings.resolve(self.path, self.owner.uuid)
+        if path.exists():
+            path.rmdir()
+        super().delete()
+

ox_utils/django/files/processors/__init__.py

@@ -0,0 +1,12 @@
+from .processor import Processor
+from .image_processor import ImageProcessor
+from .pdf_processor import PDFProcessor
+from .libreoffice_processor import LibreOfficeProcessor
+from .registry import Registry
+
+
+registry = Registry(Processor())
+""" Default registry of file processors. """
+
+
+__all__ = ("Processor", "ImageProcessor", "PDFProcessor", "LibreOfficeProcessor", "Registry", "registry")

ox_utils/django/files/processors/image_processor.py

@@ -0,0 +1,56 @@
+from __future__ import annotations
+from pathlib import Path
+
+from django.utils.translation import gettext_lazy as _
+
+
+from ..exceptions import FileTypeError
+from .processor import Processor
+
+
+__all__ = ("ImageProcessor",)
+
+
+class ImageProcessor(Processor):
+    mime_types = {
+        "image/jpeg",
+        "image/png",
+        "image/gif",
+        "image/bmp",
+        "image/webp",
+        "image/tiff",
+        "image/x-ms-bmp",
+        "image/x-pcx",
+        "image/x-tga",
+        "image/x-xbitmap",
+        "image/x-xpixmap",
+        "image/x-portable-pixmap",
+        "image/x-portable-bitmap",
+        "image/x-portable-graymap",
+        "image/x-portable-anymap",
+        "image/x-rgb",
+        "image/x-icns",
+        "image/vnd.microsoft.icon",
+        "image/avif",
+        "image/heif",
+        "image/x-exr",
+    }
+    type = "image"
+    display_type = _("Image")
+
+    def _create_preview(self, path: Path, out: Path, size: tuple[int, int]) -> bool:
+        """
+        Create thumbnail for the input image.
+        :yield: :py:class:`.exceptions.FileTypeError` when file type is not supported.
+        :yield: ``FileNotFound`` if file cannot be found
+        """
+        import PIL
+        from PIL import Image
+
+        try:
+            with Image.open(path) as im:
+                im.thumbnail(size)
+                im.save(out)
+                return True
+        except (PIL.UnidentifiedImageError, ValueError, TypeError):
+            raise FileTypeError(f"File format invalid for {path}.")