diskette 0.3.0__tar.gz

diskette-0.3.0/LICENCE.txt

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2023-2024, Emencia
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

diskette-0.3.0/MANIFEST.in

@@ -0,0 +1,13 @@
+include MANIFEST.in
+include LICENCE.txt
+include README.rst
+
+recursive-include diskette/templates *
+recursive-include diskette/static *
+recursive-include diskette/locale *
+
+recursive-exclude docs *
+recursive-exclude tests *
+recursive-exclude sandbox *
+recursive-exclude * __pycache__
+recursive-exclude * *.py[co]

diskette-0.3.0/PKG-INFO

@@ -0,0 +1,92 @@
+Metadata-Version: 2.1
+Name: diskette
+Version: 0.3.0
+Summary: Export and import Django application data and medias
+Home-page: https://github.com/emencia/diskette
+Author: Emencia
+Author-email: support@emencia.com
+License: MIT
+Project-URL: Source Code, https://github.com/emencia/diskette
+Project-URL: Issue Tracker, https://github.com/emencia/diskette/issues
+Project-URL: Changelog, https://diskette.readthedocs.io/en/latest/history.html
+Project-URL: Documentation, https://diskette.readthedocs.io/
+Keywords: Python,Django
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Natural Language :: English
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 4.0
+Classifier: Framework :: Django :: 4.1
+Classifier: Framework :: Django :: 4.2
+Requires-Python: >=3.9
+Description-Content-Type: text/x-rst
+License-File: LICENCE.txt
+Requires-Dist: Django<5.0,>=4.0
+Requires-Dist: django-sendfile2>=0.7.0
+Requires-Dist: datalookup>=1.0.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == "dev"
+Requires-Dist: pytest-django; extra == "dev"
+Requires-Dist: factory-boy; extra == "dev"
+Requires-Dist: pyquery; extra == "dev"
+Requires-Dist: freezegun; extra == "dev"
+Provides-Extra: quality
+Requires-Dist: flake8; extra == "quality"
+Requires-Dist: tox; extra == "quality"
+Provides-Extra: doc
+Requires-Dist: sphinx; extra == "doc"
+Requires-Dist: furo==2023.7.26; extra == "doc"
+Requires-Dist: sphinx-copybutton==0.5.2; extra == "doc"
+Requires-Dist: tabulate; extra == "doc"
+Requires-Dist: project-composer; extra == "doc"
+Provides-Extra: doc-live
+Requires-Dist: livereload; extra == "doc-live"
+Provides-Extra: release
+Requires-Dist: twine; extra == "release"
+
+.. _Python: https://www.python.org/
+.. _Django: https://www.djangoproject.com/
+.. _django-sendfile2: https://github.com/moggers87/django-sendfile2
+.. _datalookup: https://datalookup.readthedocs.io/
+
+========
+Diskette
+========
+
+Export and import Django application datas and medias.
+
+
+Features
+********
+
+* Based on *Django apps* to know about applications and their models;
+* Application datas are dumped with Django ``dumpdata`` command as JSON fixtures, dumps
+  can be naturally loaded in any database type using Django command  ``loaddata``;
+* Define application to be dumped with multiple options;
+* Advanced data drainage for undefined applications;
+* Media archiving is done through *Storages* (not *Django storages*) that can be
+  whatever directory you need to backup;
+* Many excluding rules for datas and storages to avoid useless content in archive;
+* Build a complete archive that can be automatically loaded with Diskette or manually;
+* Support models made with ``django-polymorphic``;
+
+
+Dependancies
+************
+
+* `Python`_>=3.9;
+* `Django`_>=4.0,<5.0;
+* `django-sendfile2`_>=0.7.0;
+* `datalookup`_>=1.0.0;
+
+
+Links
+*****
+
+* Read the documentation on `Read the docs <https://diskette.readthedocs.io/>`_;
+* Download its `PyPi package <https://pypi.python.org/pypi/diskette>`_;
+* Clone it on its `Github repository <https://github.com/emencia/diskette>`_;

diskette-0.3.0/README.rst

@@ -0,0 +1,42 @@
+.. _Python: https://www.python.org/
+.. _Django: https://www.djangoproject.com/
+.. _django-sendfile2: https://github.com/moggers87/django-sendfile2
+.. _datalookup: https://datalookup.readthedocs.io/
+
+========
+Diskette
+========
+
+Export and import Django application datas and medias.
+
+
+Features
+********
+
+* Based on *Django apps* to know about applications and their models;
+* Application datas are dumped with Django ``dumpdata`` command as JSON fixtures, dumps
+  can be naturally loaded in any database type using Django command  ``loaddata``;
+* Define application to be dumped with multiple options;
+* Advanced data drainage for undefined applications;
+* Media archiving is done through *Storages* (not *Django storages*) that can be
+  whatever directory you need to backup;
+* Many excluding rules for datas and storages to avoid useless content in archive;
+* Build a complete archive that can be automatically loaded with Diskette or manually;
+* Support models made with ``django-polymorphic``;
+
+
+Dependancies
+************
+
+* `Python`_>=3.9;
+* `Django`_>=4.0,<5.0;
+* `django-sendfile2`_>=0.7.0;
+* `datalookup`_>=1.0.0;
+
+
+Links
+*****
+
+* Read the documentation on `Read the docs <https://diskette.readthedocs.io/>`_;
+* Download its `PyPi package <https://pypi.python.org/pypi/diskette>`_;
+* Clone it on its `Github repository <https://github.com/emencia/diskette>`_;

diskette-0.3.0/diskette/__init__.py

@@ -0,0 +1,6 @@
+"""A system to compose project parts from applications enabled in a manifest"""
+from importlib.metadata import version
+
+
+__pkgname__ = "diskette"
+__version__ = version(__pkgname__)

diskette-0.3.0/diskette/apps.py

@@ -0,0 +1,7 @@
+from django.apps import AppConfig
+
+
+class DisketteConfig(AppConfig):
+    name = "diskette"
+    verbose_name = "Diskette"
+    default_auto_field = "django.db.models.AutoField"

diskette-0.3.0/diskette/contrib/composer/collectors.py

@@ -0,0 +1,9 @@
+class DisketteDefinitionsCollector:
+    """
+    Diskette definition collector for 'project-composer'.
+    """
+    def define(self):
+        return []
+
+    def get_definitions(self):
+        return [d for d in self.define()]

diskette-0.3.0/diskette/contrib/composer/processors.py

@@ -0,0 +1,23 @@
+from project_composer.processors import ClassProcessor
+
+
+class DisketteDefinitionsProcessor(ClassProcessor):
+    """
+    'project-composer' Processor to collect Diskette application definitions from
+    enabled applications.
+    """
+    def get_module_path(self, name):
+        """
+        Return a Python path for a module name.
+
+        Arguments:
+            name (string): Module name.
+
+        Returns:
+            string: Module name prefixed with repository path if it is not empty else
+            returns just the module name.
+        """
+        return "{base}.{part}".format(
+            base=self.composer.get_application_base_module_path(name),
+            part="disk",
+        )

diskette-0.3.0/diskette/contrib/django_configuration.py

@@ -0,0 +1,42 @@
+"""
+.. _django-configuration: https://django-configurations.readthedocs.io/en/stable/
+"""
+from ..settings import (
+    DISKETTE_APPS,
+    DISKETTE_STORAGES,
+    DISKETTE_STORAGES_EXCLUDES,
+    DISKETTE_DUMP_PATH,
+    DISKETTE_DUMP_FILENAME,
+    DISKETTE_LOAD_STORAGES_PATH,
+)
+
+
+class DisketteDefaultSettings:
+    """
+    Default Diskette settings class to use with a `django-configuration`_ class.
+
+    You could use it like so: ::
+
+        from configurations import Configuration
+        from diskette.contrib.django_configuration import DisketteDefaultSettings
+
+        class Dev(DisketteDefaultSettings, Configuration):
+            DEBUG = True
+
+            DISKETTE_DUMP_FILENAME = "foo.tar.gz"
+
+    This will override only the setting ``DISKETTE_DUMP_FILENAME``, all other
+    Diskette settings will have the default values from ``diskette.settings``.
+    """
+
+    DISKETTE_APPS = DISKETTE_APPS
+
+    DISKETTE_STORAGES = DISKETTE_STORAGES
+
+    DISKETTE_STORAGES_EXCLUDES = DISKETTE_STORAGES_EXCLUDES
+
+    DISKETTE_DUMP_PATH = DISKETTE_DUMP_PATH
+
+    DISKETTE_DUMP_FILENAME = DISKETTE_DUMP_FILENAME
+
+    DISKETTE_LOAD_STORAGES_PATH = DISKETTE_LOAD_STORAGES_PATH

diskette-0.3.0/diskette/core/applications/__init__.py

@@ -0,0 +1,9 @@
+from .definitions import ApplicationConfig, DrainApplicationConfig
+from .lookupstore import DjangoAppLookupStore
+
+
+__all__ = [
+    "ApplicationConfig",
+    "DjangoAppLookupStore",
+    "DrainApplicationConfig",
+]

diskette-0.3.0/diskette/core/applications/definitions.py

@@ -0,0 +1,333 @@
+from pathlib import Path
+
+from django.utils.text import slugify
+
+from ...exceptions import ApplicationConfigError
+from ..defaults import DEFAULT_FORMAT, AVAILABLE_FORMATS
+
+from .store import get_appstore
+
+
+appstore = get_appstore()
+
+
+class ApplicationConfig:
+    """
+    Application model to validate and store application details.
+
+    TODO: Another name would better to avoid mental clash with Django "AppConfig".
+    ApplicationModel (+2) ? ApplicationDataDef (0)? ApplicationDefinition (+3) ?
+    DataDefinition (+3) ?
+
+    Arguments:
+        name (string): Application name, almost anything but it may be slugified for
+            internal usages so avoid too much longer text and special characters.
+        models (list): List of labels. A label is either an application label like
+            ``auth`` or a full model label like ``auth.user``. Labels are validated,
+            they must exists in Django application registry.
+
+    Keyword Arguments:
+        filename (string): The filename to use if application dump is to be written in
+            a file. The filename also determine the format used to dump data. If you
+            want another format that the default one you will have to define
+            it from the filename even you don't plan to write dump to a file.
+            Finally if not given, the filename will be automatically defined with
+            slugified ``name`` with default format.
+        excludes (list): The list of excluded model labels that won't be collected
+            into the application dump. Currently, the excluded labels are not
+            validated.
+        natural_foreign (boolean): Enable usage of natural foreign key.
+        natural_primary (boolean): Enable usage of natural primary key.
+        comments (string): Free text not used internally.
+        allow_drain (boolean): Define if application allows its excluded models to be
+            drained. Default is ``False`` to avoid implicit draining of data that may
+            not be wanted.
+        dump_command (string): Custom dump command to use for this specific application
+            instead of default ``dumpdata``. If you have models that use
+            ``django-polymorphic`` you should give value ``polymorphic_dumpdata`` here.
+
+    Attributes:
+        is_drain (boolean): Declare application as a special drain application. This
+            should always be ``False`` for a common application, ``True`` value is
+            reserved to ``DrainApplicationConfig``.
+    """
+    CONFIG_ATTRS = [
+        "name", "models", "excludes", "retention", "natural_foreign", "natural_primary",
+        "comments", "filename", "is_drain", "allow_drain", "dump_command",
+    ]
+    OPTIONS_ATTRS = [
+        "models", "excludes", "natural_foreign", "natural_primary", "filename",
+    ]
+
+    def __init__(self, name, models=[], excludes=None, natural_foreign=False,
+                 natural_primary=False, comments=None, filename=None,
+                 is_drain=None, allow_drain=False, dump_command=None):
+        self.name = name
+        self._models = [models] if isinstance(models, str) else models
+        self._excludes = excludes or []
+        self.natural_foreign = natural_foreign
+        self.natural_primary = natural_primary
+        self.comments = comments
+        self.filename = filename or self.get_filename()
+        self.allow_drain = allow_drain
+        self.dump_command = dump_command
+        self.is_drain = False
+
+    def __repr__(self):
+        return "<{klass}: {name}>".format(
+            klass=self.__class__.__name__,
+            name=self.name
+        )
+
+    @property
+    def models(self):
+        """
+        List all fully qualified model labels to include, either implicitely and
+        explicitely from given ``models`` argument.
+
+        NOTE: models, excludes and retention attributes may be cached since they have
+        no reason to change.
+
+        Returns:
+            list: Fully qualified model labels.
+        """
+        return [
+            model.label
+            for model in appstore.get_models_inclusions(
+                self._models, excludes=self._excludes
+            )
+        ]
+
+    @property
+    def excludes(self):
+        """
+        List all fully qualified model labels to exclude, either implicitely and
+        explicitely from given ``_excludes`` argument.
+
+        Returns:
+            list: Fully qualified model labels.
+        """
+        return [
+            model.label
+            for model in appstore.get_models_exclusions(
+                self._models, excludes=self._excludes
+            )
+        ]
+
+    @property
+    def retention(self):
+        """
+        List all fully qualified model labels that are not allowed to be drained from
+        this application.
+
+        Included models are never allowed to be drained and exclusions models may be
+        allowed if ``allow_drain`` is enabled.
+
+        Returns:
+            list: Fully qualified model labels that won't be allowed to be drained.
+            This means labels from ``models`` on defaut and possibly the
+            ``excludes`` one also if application allows for drainage.
+        """
+        if not self.allow_drain:
+            return self.models + self.excludes
+
+        return self.models
+
+    def get_filename(self, format_extension=None):
+        """
+        Automatically determine filename from Application name and with a format
+        extension name.
+
+        Keyword Arguments:
+            format_extension (string): Custom extension to use if given.
+
+        Returns
+            string: Filename.
+        """
+        return slugify(self.name) + "." + (format_extension or DEFAULT_FORMAT)
+
+    def as_config(self):
+        """
+        Returns Application configuration suitable for dump history.
+
+        Keyword Arguments:
+            name (boolean): To include or not the name into the dict.
+            commented (boolean): To include or not the comments into the dict.
+
+        Returns
+            dict: Application data.
+        """
+        return {
+            name: getattr(self, name)
+            for name in self.CONFIG_ATTRS
+        }
+
+    def as_options(self):
+        """
+        Returns Application options suitable to pass to dumpdata command.
+
+        Keyword Arguments:
+            name (boolean): To include or not the name into the dict.
+            commented (boolean): To include or not the comments into the dict.
+
+        Returns
+            dict: Application data.
+        """
+        return {
+            name: getattr(self, name)
+            for name in self.OPTIONS_ATTRS
+        }
+
+    def validate_includes(self):
+        """
+        Validate include labels from ``_models`` attribute.
+        """
+        # Models are required but not for an application drain object
+        if not self._models and self.is_drain is False:
+            msg = "{obj}: 'models' must not be an empty value."
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+            ))
+
+        unknow_apps, unknow_models = appstore.check_unexisting_labels(self._models)
+
+        if len(unknow_apps) > 0:
+            msg = (
+                "{obj}: Some given app labels to include does not exists: {labels}"
+            )
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+                labels=", ".join(unknow_apps),
+            ))
+
+        if len(unknow_models) > 0:
+            msg = (
+                "{obj}: Some given models labels to include does not exists: {labels}"
+            )
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+                labels=", ".join(unknow_models),
+            ))
+
+    def validate_excludes(self):
+        """
+        Validate exclude labels from ``excludes`` attribute.
+        """
+        if not isinstance(self._excludes, list):
+            msg = "{obj}: 'excludes' argument must be a list."
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+            ))
+        else:
+            errors = appstore.is_fully_qualified_labels(self._excludes)
+            if errors:
+                msg = (
+                    "{obj}: 'excludes' argument can only contains fully qualified "
+                    "labels (like 'foo.bar') these ones are invalid: {labels}"
+                )
+                raise ApplicationConfigError(msg.format(
+                    obj=self.__repr__(),
+                    labels=", ".join(errors),
+                ))
+
+        unknow_apps, unknow_models = appstore.check_unexisting_labels(self._excludes)
+
+        if len(unknow_apps) > 0:
+            msg = (
+                "{obj}: Some given app labels to exclude does not exists: {labels}"
+            )
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+                labels=", ".join(unknow_apps),
+            ))
+
+        if len(unknow_models) > 0:
+            msg = (
+                "{obj}: Some given models labels to exclude does not exists: {labels}"
+            )
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+                labels=", ".join(unknow_models),
+            ))
+
+    def validate_filename(self):
+        # Filename must have a file extension to discover serialization format
+        extension = Path(self.filename).suffix
+        if not extension:
+            msg = (
+                "{obj}: Given file name '{filename}' must have a file extension to "
+                "discover format."
+            )
+            raise ApplicationConfigError(msg.format(
+                obj=self.__repr__(),
+                filename=self.filename,
+            ))
+        # File extension must correspond to an allowed format
+        else:
+            # Remove leading dot
+            extension = extension[1:]
+            if extension not in AVAILABLE_FORMATS:
+                msg = (
+                    "{obj}: Given file name '{filename}' must use a file extension "
+                    "from allowed formats: {formats}"
+                )
+                raise ApplicationConfigError(msg.format(
+                    obj=self.__repr__(),
+                    filename=self.filename,
+                    formats=", ".join(AVAILABLE_FORMATS),
+                ))
+
+    def validate(self):
+        """
+        Validate Application options.
+
+        Raises:
+            ApplicationConfigError: In case of invalid values from options.
+        """
+        self.validate_filename()
+        self.validate_includes()
+        self.validate_excludes()
+
+
+class DrainApplicationConfig(ApplicationConfig):
+    """
+    Special application to drain remaining models from apps.
+
+    On default a drain will dump anything that have not been defined from apps. Its
+    base goal is to dump data from undefined applications.
+
+    Attributes:
+        drain_excluded (boolean): If enabled, the drain will accept to drain exclusion
+            from applications which allow it. Else the drain will exclude also the
+            application exclusion. Default is disabled.
+    """
+    CONFIG_ATTRS = [
+        "name", "models", "excludes", "natural_foreign", "natural_primary", "comments",
+        "filename", "is_drain", "drain_excluded", "dump_command",
+    ]
+    OPTIONS_ATTRS = [
+        "models", "excludes", "natural_foreign", "natural_primary", "filename",
+    ]
+
+    def __init__(self, *args, **kwargs):
+        self.drain_excluded = kwargs.pop("drain_excluded", False)
+
+        super().__init__(*args, **kwargs)
+
+        # Drain never allow for any inclusion
+        self._models = []
+        # Force as a drain only
+        self.is_drain = True
+        # It is not allowed to be drained itself
+        self.allow_drain = False
+
+    @property
+    def excludes(self):
+        """
+        Just returns exclude labels as given since drain excludes are meaningful
+        enough.
+
+        Returns:
+            list: Fully qualified model labels.
+        """
+        return self._excludes