filedepot 0.12.0__py3-none-any.whl

depot/fields/filters/__init__.py

@@ -0,0 +1 @@
+  

depot/fields/filters/thumbnails.py

@@ -0,0 +1,54 @@
+from depot.io import utils
+from depot.manager import DepotManager
+from ..interfaces import FileFilter
+from PIL import Image
+from io import BytesIO
+
+
+class WithThumbnailFilter(FileFilter):
+    """Uploads a thumbnail together with the file.
+
+    Takes for granted that the file is an image.
+    The resulting uploaded file will provide three additional
+    properties named:
+
+        - ``thumb_X_id`` -> The depot file id
+        - ``thumb_X_path`` -> Where the file is available in depot
+        - ``thumb_X_url`` -> Where the file is served.
+
+    Where ``X`` is the resolution specified as ``size`` in the
+    filter initialization. By default this is ``(128, 128)``?so
+    you will get ``thumb_128x128_id``, ``thumb_128x128_url`` and
+    so on.
+
+    .. warning::
+
+        Requires Pillow library
+
+    """
+    def __init__(self, size=(128,128), format='PNG'):
+        self.thumbnail_size = size
+        self.thumbnail_format = format
+
+    def on_save(self, uploaded_file):
+        close_content, content = utils.file_from_content(uploaded_file.original_content)
+
+        try:
+            thumbnail = Image.open(content)
+            thumbnail.thumbnail(self.thumbnail_size, Image.BILINEAR)
+            thumbnail = thumbnail.convert('RGBA')
+            thumbnail.format = self.thumbnail_format
+
+            output = BytesIO()
+            thumbnail.save(output, self.thumbnail_format)
+            output.seek(0)
+
+            thumb_name = 'thumb_%sx%s' % self.thumbnail_size
+            thumb_file_name = '%s.%s' % (thumb_name, self.thumbnail_format.lower())
+            thumb_path, thumb_id = uploaded_file.store_content(output, thumb_file_name)
+            uploaded_file[thumb_name + '_id'] = thumb_id
+            uploaded_file[thumb_name + '_path'] = thumb_path
+            uploaded_file[thumb_name + '_url'] = DepotManager.get_middleware().url_for(thumb_path)
+        finally:
+            if close_content:
+                content.close()

depot/fields/interfaces.py

@@ -0,0 +1,108 @@
+from abc import ABCMeta, abstractmethod
+from depot.manager import DepotManager
+
+
+class FileFilter(object, metaclass=ABCMeta):
+    """Interface that must be implemented by file filters.
+
+    File filters get executed whenever a file is stored on the database
+    using one of the supported fields. Can be used to add additional data
+    to the stored file or change it. When file filters are run the file
+    has already been stored.
+
+    """
+    @abstractmethod
+    def on_save(self, uploaded_file):  # pragma: no cover
+        """Filters are required to provide their own implementation"""
+        return
+
+
+class DepotFileInfo(dict, metaclass=ABCMeta):
+    """Keeps information on a content related to a specific depot.
+
+    By itself the DepotFileInfo does nothing, it is required to implement
+    a :meth:`process_content` method that actually saves inside the
+    file info the information related to the content. The only information
+    which is saved by default is the depot name itself.
+
+    It is a specialized dictionary that provides also attribute style access,
+    the dictionary parent permits easy encoding/decoding to most marshalling
+    systems.
+
+    """
+    def __init__(self, content, depot_name=None):
+        super(DepotFileInfo, self).__init__()
+        self._thaw()
+
+        if isinstance(content, dict):
+            object.__setattr__(self, 'original_content', None)
+            self.update(content)
+        else:
+            object.__setattr__(self, 'original_content', content)
+            if depot_name is None:
+                depot_name = DepotManager.get_default()
+
+            depot_name = DepotManager.resolve_alias(depot_name)
+            if not depot_name:
+                raise ValueError('Storage has not been found in DEPOT')
+
+            self['depot_name'] = depot_name
+            self['files'] = []
+            self.process_content(content)
+
+        self._freeze()
+
+    @abstractmethod
+    def process_content(self, content, filename=None, content_type=None):  # pragma: no cover
+        """Process content in the given depot.
+
+        This is implemented by subclasses to provide some kind of behaviour on the
+        content in the related Depot. The default implementation is provided by
+        :class:`depot.fields.upload.UploadedFile` which stores the content into
+        the depot.
+
+        """
+        return
+
+    def __getitem__(self, key):
+        return dict.__getitem__(self, key)
+
+    def __getattr__(self, name):
+        try:
+            return self[name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __setitem__(self, key, value):
+        if object.__getattribute__(self, '_frozen'):
+            raise TypeError('Already saved files are immutable')
+        return dict.__setitem__(self, key, value)
+
+    __setattr__ = __setitem__
+
+    def __delattr__(self, name):
+        if object.__getattribute__(self, '_frozen'):
+            raise TypeError('Already saved files are immutable')
+
+        try:
+            del self[name]
+        except KeyError:
+            raise AttributeError(name)
+
+    def __delitem__(self, key):
+        if object.__getattribute__(self, '_frozen'):
+            raise TypeError('Already saved files are immutable')
+        dict.__delitem__(self, key)
+
+    def _apply_filters(self, filters):
+        if self.original_content is not None:
+            self._thaw()
+            for filt in filters:
+                filt.on_save(self)
+            self._freeze()
+
+    def _freeze(self):
+        object.__setattr__(self, '_frozen', True)
+
+    def _thaw(self):
+        object.__setattr__(self, '_frozen', False)

depot/fields/ming.py

@@ -0,0 +1,168 @@
+from ming.odm import mapper
+from ming.odm.property import FieldProperty
+from ming.odm.base import session, state, ObjectState
+from ming.odm.odmsession import SessionExtension
+from ming.schema import Anything
+
+from depot.manager import DepotManager
+from .upload import UploadedFile
+
+
+class _UploadedFileSchema(Anything):
+    pass
+
+
+class UploadedFileProperty(FieldProperty):
+    """Provides support for storing attachments to **Ming** MongoDB models.
+
+    ``UploadedFileProperty`` can be used as a field type to store files
+    into the model. The actual file itself will be uploaded to the
+    default Storage, and only the :class:`depot.fields.upload.UploadedFile`
+    information will be stored on the database.
+
+    The ``UploadedFileProperty`` is UnitOfWork aware, so it will delete
+    every uploaded file whenever unit of work is flushed and deletes a Document
+    that stored files or changes the field of a document storing files. This is
+    the reason you should never associate the same :class:`depot.fields.upload.UploadedFile`
+    to two different ``UploadedFileProperty``, otherwise you might delete a file
+    already used by another document. It is usually best to just set the ``file``
+    or ``bytes`` as content of the column and let the ``UploadedFileProperty``
+    create the :class:`depot.fields.upload.UploadedFile` by itself whenever it's content is set.
+
+    .. warning::
+
+        As the Ming UnitOfWork does not notify any event in case it gets cleared instead
+        of being flushed all the files uploaded before clearing the unit of work will be
+        already uploaded but won't have a document referencing them anymore, so DEPOT will
+        be unable to delete them for you.
+
+    """
+    def __init__(self,  filters=tuple(), upload_type=UploadedFile, upload_storage=None):
+        FieldProperty.__init__(self, _UploadedFileSchema())
+        self._filters = filters
+        self._upload_type = upload_type
+        self._upload_storage = upload_storage
+
+    def __set__(self, instance, value):
+        if value is not None and not isinstance(value, UploadedFile):
+            upload_type = self._upload_type
+            value = upload_type(value, self._upload_storage)
+
+        if isinstance(value, UploadedFile):
+            value._apply_filters(self._filters)
+
+        old_value = self.__get__(instance, instance.__class__)
+        DepotExtension.get_depot_history(instance).swap(old_value, value)
+        return FieldProperty.__set__(self, instance, value)
+
+    def __get__(self, instance, owner=None):
+        try:
+            value = FieldProperty.__get__(self, instance, owner)
+        except AttributeError:
+            value = None
+
+        if not value:
+            return None
+
+        if instance is None:
+            return value
+
+        return self._upload_type(value)
+
+    """
+    # Looks like this should do nothing on ming.
+    def __delete__(self, instance, owner=None):
+        old_value = self.__get__(instance, instance.__class__)
+        DepotExtension.get_depot_history(instance).delete(old_value)
+        return FieldProperty.__delete__(self, instance, owner)
+    """
+
+
+class DepotExtension(SessionExtension):
+    """Extends the Ming Session to track files.
+
+    Deletes old files when an entry gets removed or replaced,
+    apply this as a Ming ``SessionExtension`` according to Ming
+    documentation.
+    """
+    @classmethod
+    def get_depot_history(cls, instance):
+        istate = state(instance)
+        if not hasattr(istate, '_depot_history'):
+            istate._depot_history = _DepotHistory()
+        return istate._depot_history
+
+    def _check_object_deleted(self, obj):
+        hist = self.get_depot_history(obj)
+        if state(obj).status == ObjectState.deleted:
+            for prop in mapper(obj).properties:
+                if isinstance(prop, UploadedFileProperty):
+                    current_value = prop.__get__(obj, obj.__class__)
+                    hist.delete(current_value)
+            self._flush_object(obj)
+
+    def _flush_object(self, obj):
+        history = self.get_depot_history(obj)
+        for entry in history.deleted:
+            depot, fileid = entry.split('/', 1)
+            depot = DepotManager.get(depot)
+            depot.delete(fileid)
+        history.clear()
+
+    def before_flush(self, obj=None):
+        if obj:
+            self._check_object_deleted(obj)
+        else:
+            for class_, id_, obj in self.session.imap:
+                self._check_object_deleted(obj)
+
+    def after_flush(self, obj=None):
+        if obj:
+            self._flush_object(obj)
+        else:
+            for class_, id_, obj in self.session.imap:
+                self._flush_object(obj)
+
+
+class _DepotHistory(object):
+    def __init__(self):
+        self.clear()
+
+    def _extract_files(self, obj):
+        return obj['files']
+
+    def add(self, obj):
+        if obj is None:
+            return
+
+        files = self._extract_files(obj)
+        self.deleted.difference_update(obj)
+        self.new.update(files)
+
+    def delete(self, obj):
+        if obj is None:
+            return
+
+        files = self._extract_files(obj)
+        self.new.difference_update(obj)
+        self.deleted.update(files)
+
+    def swap(self, old, new):
+        self.delete(old)
+        self.add(new)
+
+    def clear(self):
+        self.deleted = set()
+        self.new = set()
+
+
+try:  # pragma: no cover
+    from sprox.mg.widgetselector import MingWidgetSelector
+    from tw2.forms import FileField as TW2FileField
+    MingWidgetSelector.default_widgets.setdefault(_UploadedFileSchema, TW2FileField)
+
+    from sprox.mg.validatorselector import MingValidatorSelector
+    from ..validators import TW2FileIntentValidator
+    MingValidatorSelector.default_validators.setdefault(_UploadedFileSchema, TW2FileIntentValidator)
+except ImportError:  # pragma: no cover
+    pass

depot/fields/specialized/image.py

@@ -0,0 +1,76 @@
+from depot.io import utils
+from depot.manager import DepotManager
+from ..upload import UploadedFile
+from depot.io.interfaces import FileStorage
+from PIL import Image
+from depot.io.utils import INMEMORY_FILESIZE
+from tempfile import SpooledTemporaryFile
+
+
+class UploadedImageWithThumb(UploadedFile):
+    """Uploads an Image with thumbnail.
+
+    The default thumbnail format and size are PNG@128x128, those can be changed
+    by inheriting the ``UploadedImageWithThumb`` and replacing the
+    ``thumbnail_format`` and ``thumbnail_size`` class properties.
+
+    The Thumbnail file is accessible as ``.thumb_file`` while the
+    thumbnail url is ``.thumb_url``.
+
+    .. warning::
+
+        Requires Pillow library
+
+    """
+
+    max_size = 1024
+    thumbnail_format = 'PNG'
+    thumbnail_size = (128, 128)
+
+    def process_content(self, content, filename=None, content_type=None):
+        orig_content = content
+        temporary_content = False
+        __, filename, content_type = FileStorage.fileinfo(orig_content)
+        close_content, content = utils.file_from_content(content)
+
+        try:
+            uploaded_image = Image.open(content)
+            if max(uploaded_image.size) >= self.max_size:
+                uploaded_image.thumbnail((self.max_size, self.max_size), Image.BILINEAR)
+                content = SpooledTemporaryFile(INMEMORY_FILESIZE)
+                close_content = True
+                uploaded_image.save(content, uploaded_image.format)
+
+            content.seek(0)
+            super(UploadedImageWithThumb, self).process_content(content, filename, content_type)
+        finally:
+            if close_content:
+                content.close()
+
+        thumbnail = uploaded_image.copy()
+        thumbnail.thumbnail(self.thumbnail_size, Image.LANCZOS)
+        thumbnail = thumbnail.convert('RGBA')
+        thumbnail.format = self.thumbnail_format
+
+        with SpooledTemporaryFile(INMEMORY_FILESIZE) as output:
+            thumbnail.save(output, self.thumbnail_format)
+            output.seek(0)
+
+            thumb_path, thumb_id = self.store_content(output,
+                                                    'thumb.%s' % self.thumbnail_format.lower())
+            self['thumb_id'] = thumb_id
+            self['thumb_path'] = thumb_path
+
+            thumbnail_file = self.thumb_file
+            self['_thumb_public_url'] = thumbnail_file.public_url
+
+    @property
+    def thumb_file(self):
+        return self.depot.get(self.thumb_id)
+
+    @property
+    def thumb_url(self):
+        public_url = self['_thumb_public_url']
+        if public_url:
+            return public_url
+        return DepotManager.get_middleware().url_for(self['thumb_path'])

depot/fields/sqlalchemy.py

@@ -0,0 +1,193 @@
+import itertools
+
+from sqlalchemy import types
+from sqlalchemy import event
+from sqlalchemy import orm
+from sqlalchemy import inspect
+from sqlalchemy.orm import ColumnProperty
+from sqlalchemy.orm import Session
+from sqlalchemy.orm.attributes import get_history
+
+from depot.manager import DepotManager
+from .upload import UploadedFile
+
+
+class UploadedFileField(types.TypeDecorator):
+    """Provides support for storing attachments to **SQLAlchemy** models.
+
+    ``UploadedFileField`` can be used as a Column type to store files
+    into the model. The actual file itself will be uploaded to the
+    default Storage, and only the :class:`depot.fields.upload.UploadedFile`
+    information will be stored on the database.
+
+    The ``UploadedFileField`` is transaction aware, so it will delete
+    every uploaded file whenever the transaction is rolled back and will
+    delete any old file whenever the transaction is committed. This is
+    the reason you should never associate the same :class:`depot.fields.upload.UploadedFile`
+    to two different ``UploadedFileField``, otherwise you might delete a file
+    already used by another model. It is usually best to just set the ``file``
+    or ``bytes`` as content of the column and let the ``UploadedFileField``
+    create the :class:`depot.fields.upload.UploadedFile` by itself whenever it's content is set.
+
+    """
+    impl = types.Unicode
+    cache_ok = False
+
+    def __init__(self, filters=tuple(), upload_type=UploadedFile, upload_storage=None, *args, **kw):
+        super(UploadedFileField, self).__init__(*args, **kw)
+        self._filters = filters
+        self._upload_type = upload_type
+        self._upload_storage = upload_storage
+
+    def load_dialect_impl(self, dialect):
+        return dialect.type_descriptor(types.VARCHAR(4000))
+
+    def process_bind_param(self, value, dialect):
+        if not value:
+            return None
+
+        if not isinstance(value, self._upload_type):
+            raise ValueError('AttachmentField requires %s, '
+                             'got %s instead' % (self._upload_type, type(value)))
+
+        return value.encode()
+
+    def process_result_value(self, value, dialect):
+        if not value:
+            return None
+        return self._upload_type.decode(value)
+
+
+class _SQLAMutationTracker(object):
+    mapped_entities = {}
+
+    @classmethod
+    def _field_set(cls, target, value, oldvalue, initiator):
+        if value is None or isinstance(value, UploadedFile):
+            return value
+
+        inspection = inspect(target)
+        set_property = inspection.mapper.get_property(initiator.key)
+        column_type = set_property.columns[0].type
+        assert(isinstance(column_type, UploadedFileField))
+
+        upload_type = column_type._upload_type
+        value = upload_type(value, column_type._upload_storage)
+        value._apply_filters(column_type._filters)
+
+        # Mark newly added files on assignment, so that in case of a rollback
+        # without going through flush we have track of the files
+        # that need to be rolled back.
+        session = inspection.session
+        if session is not None:
+            session._depot_new = getattr(session, '_depot_new', set())
+            session._depot_new.update(value.files)
+        else:
+            # If object is not attached to a session preserve the files
+            # until it gets added to one.
+            target._depot_new = getattr(target, '_depot_new', set())
+            target._depot_new.update(value.files)
+
+        return value
+
+    @classmethod
+    def _mapper_configured(cls, mapper, class_):
+        for mapper_property in mapper.iterate_properties:
+            if isinstance(mapper_property, ColumnProperty):
+                for idx, col in enumerate(mapper_property.columns):
+                    if isinstance(col.type, UploadedFileField):
+                        if idx > 0:
+                            # Not clear when this might happen, but ColumnProperty can have
+                            # multiple columns assigned. We should probably take the first one
+                            # as is done by ColumnProperty.expression but there is no guarantee
+                            # that it would be the right thing to do.
+                            raise TypeError('UploadedFileField currently supports a single column')
+                        cls.mapped_entities.setdefault(class_, []).append(mapper_property.key)
+                        event.listen(mapper_property, 'set', cls._field_set, retval=True, propagate=True)
+
+    @classmethod
+    def _session_rollback(cls, session, previous_transaction):
+        if hasattr(session, '_depot_old'):
+            del session._depot_old
+        if hasattr(session, '_depot_new'):
+            for entry in session._depot_new:
+                depot, fileid = entry.split('/', 1)
+                depot = DepotManager.get(depot)
+                depot.delete(fileid)
+            del session._depot_new
+
+    @classmethod
+    def _session_committed(cls, session):
+        if hasattr(session, '_depot_old'):
+            for entry in session._depot_old:
+                depot, fileid = entry.split('/', 1)
+                depot = DepotManager.get(depot)
+                depot.delete(fileid)
+            del session._depot_old
+        if hasattr(session, '_depot_new'):
+            del session._depot_new
+
+    @classmethod
+    def _session_flush(cls, session, flush_context, instances):
+        for obj in session.deleted:
+            class_ = obj.__class__
+            tracked_columns = cls.mapped_entities.get(class_, tuple())
+            for col in tracked_columns:
+                value = getattr(obj, col)
+                if value is not None:
+                    session._depot_old = getattr(session, '_depot_old', set())
+                    session._depot_old.update(value.files)
+
+        for obj in session.new.union(session.dirty):
+            class_ = obj.__class__
+            tracked_columns = cls.mapped_entities.get(class_, tuple())
+            for col in tracked_columns:
+                history = get_history(obj, col)
+                added_files = itertools.chain(*(f.files for f in history.added
+                                               if f is not None))
+                deleted_files = itertools.chain(*(f.files for f in history.deleted
+                                                 if f is not None))
+
+                session._depot_new = getattr(session, '_depot_new', set())
+                session._depot_new.update(added_files)
+                session._depot_old = getattr(session, '_depot_old', set())
+                session._depot_old.update(deleted_files)
+
+    @classmethod
+    def _session_deleted_obj(cls, session, obj):
+        # Tracking indirectly deleted objects
+        # a relationship.remove, or a query().filter().delete()
+        class_ = obj.__class__
+        tracked_columns = cls.mapped_entities.get(class_, tuple())
+        for col in tracked_columns:
+            value = getattr(obj, col)
+            if value is not None:
+                session._depot_old = getattr(session, '_depot_old', set())
+                session._depot_old.update(value.files)
+
+    @classmethod
+    def _session_attached(cls, session, instance):
+        session._depot_new = getattr(session, '_depot_new', set())
+        session._depot_new.update(getattr(instance, '_depot_new', set()))
+
+    @classmethod
+    def setup(cls):
+        event.listen(orm.Mapper, 'mapper_configured', cls._mapper_configured)
+        event.listen(Session, 'after_soft_rollback', cls._session_rollback)
+        event.listen(Session, 'after_commit', cls._session_committed)
+        event.listen(Session, 'before_attach', cls._session_attached)
+        event.listen(Session, 'before_flush', cls._session_flush)
+        event.listen(Session, 'persistent_to_deleted', cls._session_deleted_obj)
+
+_SQLAMutationTracker.setup()
+
+try:  # pragma: no cover
+    from sprox.sa.widgetselector import SAWidgetSelector
+    from tw2.forms import FileField as TW2FileField
+    SAWidgetSelector.default_widgets.setdefault(UploadedFileField, TW2FileField)
+
+    from sprox.sa.validatorselector import SAValidatorSelector
+    from ..validators import TW2FileIntentValidator
+    SAValidatorSelector.default_validators.setdefault(UploadedFileField, TW2FileIntentValidator)
+except ImportError:  # pragma: no cover
+    pass

depot/fields/upload.py

@@ -0,0 +1,73 @@
+from depot.manager import DepotManager
+from .interfaces import DepotFileInfo
+import json
+
+
+class UploadedFile(DepotFileInfo):
+    """Simple :class:`depot.fields.interfaces.DepotFileInfo` implementation that stores files.
+
+    Takes a file as content and uploads it to the depot while saving around
+    most file information. Pay attention that if the file gets replaced
+    through depot manually the ``UploadedFile`` will continue to have the old data.
+
+    Also provides support for encoding/decoding using JSON for storage inside
+    databases as a plain string.
+
+    Default attributes provided for all ``UploadedFile`` include:
+        - filename     - This is the name of the uploaded file
+        - file_id      - This is the ID of the uploaded file
+        - path         - This is a depot_name/file_id path which can
+                         be used with :meth:`DepotManager.get_file` to retrieve the file
+        - content_type - This is the content type of the uploaded file
+        - uploaded_at  - This is the upload date in YYYY-MM-DD HH:MM:SS format
+        - url          - Public url of the uploaded file
+        - file         - The :class:`depot.io.interfaces.StoredFile` instance of the stored file
+    """
+    def process_content(self, content, filename=None, content_type=None):
+        """Standard implementation of :meth:`.DepotFileInfo.process_content`
+
+        This is the standard depot implementation of files upload, it will
+        store the file on the default depot and will provide the standard
+        attributes.
+
+        Subclasses will need to call this method to ensure the standard
+        set of attributes is provided.
+        """
+
+        file_path, file_id = self.store_content(content, filename, content_type)
+        self['file_id'] = file_id
+        self['path'] = file_path
+
+        saved_file = self.file
+        self['filename'] = saved_file.filename
+        self['content_type'] = saved_file.content_type
+        self['uploaded_at'] = saved_file.last_modified.strftime('%Y-%m-%d %H:%M:%S')
+        self['_public_url'] = saved_file.public_url
+
+    def store_content(self, content, filename=None, content_type=None):
+        file_id = self.depot.create(content, filename, content_type)
+        file_path = '%s/%s' % (self.depot_name, file_id)
+        self.files.append(file_path)
+        return file_path, file_id
+
+    def encode(self):
+        return json.dumps(self)
+
+    @classmethod
+    def decode(cls, data):
+        return cls(json.loads(data))
+
+    @property
+    def url(self):
+        public_url = self['_public_url']
+        if public_url:
+            return public_url
+        return DepotManager.get_middleware().url_for(self['path'])
+
+    @property
+    def depot(self):
+        return DepotManager.get(self.depot_name)
+
+    @property
+    def file(self):
+        return self.depot.get(self.file_id)

depot/io/__init__.py

@@ -0,0 +1 @@
+