django-sync-env 0.2.0__tar.gz

django_sync_env-0.2.0/PKG-INFO

@@ -0,0 +1,65 @@
+Metadata-Version: 2.1
+Name: django-sync-env
+Version: 0.2.0
+Summary: Backup, Sync and Restore Databases and Media
+Author: Dan Brosnan
+Author-email: dan.brosnan@octave.nz
+Requires-Python: >=3.10,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: django-storages (>=1.14.2,<2.0.0)
+Requires-Dist: inquirer (>=3.1.3,<4.0.0)
+Description-Content-Type: text/markdown
+
+## Django sync env
+
+django-sync-env is a Django app to manage backing up and restoring django databases and media assets easy.
+
+Detailed documentation is in the "docs" directory. (needs some work)
+
+
+## Requirements:
+- configurable backups and restore which can be ran on demand, celery cron or in dev.
+- support s3 bucket and file storage options first, add azure blob storage later
+- interactive prompts for local dev
+- command line options with --no-input for CICD and automations
+- restore latest option for media and backups for X env
+- backup before restore option
+
+
+## TODO:
+- update docs around how to development this package
+- remove command options which we wont use
+- make sure all management commands work will good error logging and handling
+
+
+Quick start
+-----------
+
+1. Add "django_sync_env" to your INSTALLED_APPS setting like this:
+
+```
+   INSTALLED_APPS = [
+   ...,
+   "django_sync_env",
+   ]
+```
+
+2. Configure the app via a settings file .typically `./settings/partial/sync_env.py` for base configuration,
+   don't forget to import this file via your base.py settings file
+   And override any required settings per environment via `./settings/partials/[env].py`
+   see [example_partials](docs/example_partials.md)
+
+There is a [s3 terraform example](docs/example_terraform_aws_s3_bucket.md) for provisioning
+an aws s3 bucket, iam user, roles and policy to allow for remote backup/restore to/from a secure s3 bucket.
+
+See [management commands](docs/management_commands.md) for more details for each of the commands available.
+
+- `./manage.py sync_env_backup_db`
+- `./manage.py sync_env_backup_media`
+- `./manage.py sync_env_restore_db`
+- `./manage.py sync_env_restore_media`
+- `./manage.py sync_env_list_backups`
+

django_sync_env-0.2.0/README.md

@@ -0,0 +1,49 @@
+## Django sync env
+
+django-sync-env is a Django app to manage backing up and restoring django databases and media assets easy.
+
+Detailed documentation is in the "docs" directory. (needs some work)
+
+
+## Requirements:
+- configurable backups and restore which can be ran on demand, celery cron or in dev.
+- support s3 bucket and file storage options first, add azure blob storage later
+- interactive prompts for local dev
+- command line options with --no-input for CICD and automations
+- restore latest option for media and backups for X env
+- backup before restore option
+
+
+## TODO:
+- update docs around how to development this package
+- remove command options which we wont use
+- make sure all management commands work will good error logging and handling
+
+
+Quick start
+-----------
+
+1. Add "django_sync_env" to your INSTALLED_APPS setting like this:
+
+```
+   INSTALLED_APPS = [
+   ...,
+   "django_sync_env",
+   ]
+```
+
+2. Configure the app via a settings file .typically `./settings/partial/sync_env.py` for base configuration,
+   don't forget to import this file via your base.py settings file
+   And override any required settings per environment via `./settings/partials/[env].py`
+   see [example_partials](docs/example_partials.md)
+
+There is a [s3 terraform example](docs/example_terraform_aws_s3_bucket.md) for provisioning
+an aws s3 bucket, iam user, roles and policy to allow for remote backup/restore to/from a secure s3 bucket.
+
+See [management commands](docs/management_commands.md) for more details for each of the commands available.
+
+- `./manage.py sync_env_backup_db`
+- `./manage.py sync_env_backup_media`
+- `./manage.py sync_env_restore_db`
+- `./manage.py sync_env_restore_media`
+- `./manage.py sync_env_list_backups`

django_sync_env-0.2.0/pyproject.toml

@@ -0,0 +1,16 @@
+[tool.poetry]
+name = "django-sync-env"
+version = "0.2.0"
+description = "Backup, Sync and Restore Databases and Media"
+authors = ["Dan Brosnan <dan.brosnan@octave.nz>", "Will Cook <will@octave.nz>"]
+readme = "README.md"
+packages = [{ include = "django_sync_env", from = "src" }]
+
+[tool.poetry.dependencies]
+python = "^3.10"
+inquirer = "^3.1.3"
+django-storages = "^1.14.2"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

django_sync_env-0.2.0/src/django_sync_env/apps.py

@@ -0,0 +1,20 @@
+"""Apps for SyncEnv"""
+
+from django.apps import AppConfig
+from django.utils.translation import gettext_lazy
+
+from django_sync_env import log
+
+
+class SyncEnvConfig(AppConfig):
+    """
+    Config for SyncEnv
+    """
+
+    name = "django_sync_env"
+    label = "django_sync_env"
+    verbose_name = gettext_lazy("sync env")
+    default_auto_field = "django.db.models.AutoField"
+
+    def ready(self):
+        log.load_logger()

django_sync_env-0.2.0/src/django_sync_env/checks.py

@@ -0,0 +1,39 @@
+import re
+
+from django.core.checks import Tags, Warning, register
+
+from syncenv import settings
+
+W001 = Warning(
+    "SYNC_ENV_BACKUP_CONFIG setting has not be configured",
+    hint="Set up settings.SYNC_ENV_BACKUP_CONFIG see syncenv readme",
+    id="syncenv.W001",
+)
+
+W002 = Warning(
+    "SYNC_ENV_RESTORE_CONFIG setting has not be configured",
+    hint="Set up settings.SYNC_ENV_RESTORE_CONFIG see syncenv readme",
+    id="syncenv.W002",
+)
+
+W005 = Warning(
+    "Invalid DATE_FORMAT parameter",
+    hint="settings.SYNC_ENV_DATE_FORMAT can contain only [A-Za-z0-9%_-]",
+    id="syncenv.W005",
+)
+
+
+@register(Tags.compatibility)
+def check_settings(app_configs, **kwargs):
+    errors = []
+
+    if not settings.SYNC_ENV_BACKUP_CONFIG:
+        errors.append(W001)
+
+    if not settings.SYNC_ENV_RESTORE_CONFIG:
+        errors.append(W001)
+
+    if re.search(r"[^A-Za-z0-9%_-]", settings.DATE_FORMAT):
+        errors.append(W005)
+
+    return errors

django_sync_env-0.2.0/src/django_sync_env/constants.py

@@ -0,0 +1,3 @@
+BACKUP_DATABASE_MANAGEMENT_COMMAND_NAME = 'sync_env_backup_db'
+BACKUP_LISTING_MANAGEMENT_COMMAND_NAME = 'sync_env_list_db_backups'
+BACKUP_MEDIA_MANAGEMENT_COMMAND_NAME = 'sync_env_backup_media'

django_sync_env-0.2.0/src/django_sync_env/db/base.py

@@ -0,0 +1,186 @@
+"""
+Base database connectors
+"""
+import logging
+import os
+import shlex
+from importlib import import_module
+from subprocess import Popen
+from tempfile import SpooledTemporaryFile
+
+from django.core.files.base import File
+
+from django_sync_env import settings, utils
+
+from django_sync_env.db import exceptions
+
+logger = logging.getLogger("sync_env")
+logger.setLevel(logging.DEBUG)
+
+CONNECTOR_MAPPING = {
+    # "django.db.backends.sqlite3": "syncenv.db.sqlite.SqliteConnector",
+    # "django.db.backends.mysql": "syncenv.db.mysql.MysqlDumpConnector",
+    "django.db.backends.postgresql": "django_sync_env.db.postgresql.PgDumpBinaryConnector",
+    "django.db.backends.postgresql_psycopg2": "django_sync_env.db.postgresql.PgDumpBinaryConnector",
+    # "django.db.backends.oracle": None,
+    # "django_mongodb_engine": "syncenv.db.mongodb.MongoDumpConnector",
+    # "djongo": "syncenv.db.mongodb.MongoDumpConnector",
+    # "django.contrib.gis.db.backends.postgis": "syncenv.db.postgresql.PgDumpGisConnector",
+    # "django.contrib.gis.db.backends.mysql": "syncenv.db.mysql.MysqlDumpConnector",
+    # "django.contrib.gis.db.backends.oracle": None,
+    # "django.contrib.gis.db.backends.spatialite": "syncenv.db.sqlite.SqliteConnector",
+    # "django_prometheus.db.backends.postgresql": "syncenv.db.postgresql.PgDumpBinaryConnector",
+    # "django_prometheus.db.backends.sqlite3": "syncenv.db.sqlite.SqliteConnector",
+    # "django_prometheus.db.backends.mysql": "syncenv.db.mysql.MysqlDumpConnector",
+    # "django_prometheus.db.backends.postgis": "syncenv.db.postgresql.PgDumpGisConnector",
+}
+
+if settings.CUSTOM_CONNECTOR_MAPPING:
+    CONNECTOR_MAPPING.update(settings.CUSTOM_CONNECTOR_MAPPING)
+
+
+def get_connector(database_name=None):
+    """
+    Get a connector from its database key in settings.
+    """
+    from django.db import DEFAULT_DB_ALIAS, connections
+
+    # Get DB
+    database_name = database_name or DEFAULT_DB_ALIAS
+    connection = connections[database_name]
+    engine = connection.settings_dict["ENGINE"]
+    connector_settings = settings.CONNECTORS.get(database_name, {})
+    connector_path = connector_settings.get("CONNECTOR", CONNECTOR_MAPPING[engine])
+    connector_module_path = ".".join(connector_path.split(".")[:-1])
+    print("DEBUG connector_module_path: ", connector_module_path)
+    module = import_module(connector_module_path)
+    connector_name = connector_path.split(".")[-1]
+    connector = getattr(module, connector_name)
+    return connector(database_name, **connector_settings)
+
+
+class BaseDBConnector:
+    """
+    Base class for create database connector. This kind of object creates
+    interaction with database and allow backup and restore operations.
+    """
+
+    extension = "dump"
+    exclude = []
+
+    def __init__(self, database_name=None, **kwargs):
+        from django.db import DEFAULT_DB_ALIAS, connections
+
+        self.database_name = database_name or DEFAULT_DB_ALIAS
+        self.connection = connections[self.database_name]
+        for attr, value in kwargs.items():
+            setattr(self, attr.lower(), value)
+
+    @property
+    def settings(self):
+        """Mix of database and connector settings."""
+        if not hasattr(self, "_settings"):
+            sett = self.connection.settings_dict.copy()
+            sett.update(settings.CONNECTORS.get(self.database_name, {}))
+            self._settings = sett
+        return self._settings
+
+    def generate_filename(self, environment, server_name=None, database_name=None):
+        return utils.filename_generate(
+            self.extension,
+            database_name=database_name,
+            environment=environment
+        )
+
+    def create_dump(self):
+        return self._create_dump()
+
+    def _create_dump(self):
+        """
+        Override this method to define dump creation.
+        """
+        raise NotImplementedError("_create_dump not implemented")
+
+    def restore_dump(self, dump):
+        """
+        :param dump: Dump file
+        :type dump: file
+        """
+        return self._restore_dump(dump)
+
+    def _restore_dump(self, dump):
+        """
+        Override this method to define dump creation.
+        :param dump: Dump file
+        :type dump: file
+        """
+        raise NotImplementedError("_restore_dump not implemented")
+
+
+class BaseCommandDBConnector(BaseDBConnector):
+    """
+    Base class for create database connector based on command line tools.
+    """
+
+    dump_prefix = ""
+    dump_suffix = ""
+    restore_prefix = ""
+    restore_suffix = ""
+
+    use_parent_env = True
+    env = {}
+    dump_env = {}
+    restore_env = {}
+
+    def run_command(self, command, stdin=None, env=None):
+        """
+        Launch a shell command line.
+
+        :param command: Command line to launch
+        :type command: str
+        :param stdin: Standard input of command
+        :type stdin: file
+        :param env: Environment variable used in command
+        :type env: dict
+        :return: Standard output of command
+        :rtype: file
+        """
+        logger.debug(command)
+        cmd = shlex.split(command)
+        stdout = SpooledTemporaryFile(
+            max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR
+        )
+        stderr = SpooledTemporaryFile(
+            max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR
+        )
+        full_env = os.environ.copy() if self.use_parent_env else {}
+        full_env.update(self.env)
+        full_env.update(env or {})
+        try:
+            if isinstance(stdin, File):
+                process = Popen(
+                    cmd,
+                    stdin=stdin.open("rb"),
+                    stdout=stdout,
+                    stderr=stderr,
+                    env=full_env,
+                )
+            else:
+                process = Popen(
+                    cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env
+                )
+            process.wait()
+            if process.poll():
+                stderr.seek(0)
+                raise exceptions.CommandConnectorError(
+                    "Error running: {}\n{}".format(
+                        command, stderr.read().decode("utf-8")
+                    )
+                )
+            stdout.seek(0)
+            stderr.seek(0)
+            return stdout, stderr
+        except OSError as err:
+            raise exceptions.CommandConnectorError(
+                f"Error running: {command}\n{str(err)}"
+            )

django_sync_env-0.2.0/src/django_sync_env/db/exceptions.py

@@ -0,0 +1,17 @@
+"""Exceptions for database connectors."""
+
+
+class ConnectorError(Exception):
+    """Base connector error"""
+
+
+class DumpError(ConnectorError):
+    """Error on dump"""
+
+
+class RestoreError(ConnectorError):
+    """Error on restore"""
+
+
+class CommandConnectorError(ConnectorError):
+    """Failing command"""

django_sync_env-0.2.0/src/django_sync_env/db/mongodb.py

@@ -0,0 +1,54 @@
+from syncenv import utils
+
+from .base import BaseCommandDBConnector
+
+
+class MongoDumpConnector(BaseCommandDBConnector):
+    """
+    MongoDB connector, creates dump with ``mongodump`` and restore with
+    ``mongorestore``.
+    """
+
+    dump_cmd = "mongodump"
+    restore_cmd = "mongorestore"
+    object_check = True
+    drop = True
+
+    def _create_dump(self):
+        cmd = f"{self.dump_cmd} --db {self.settings['NAME']}"
+        host = self.settings.get("HOST") or "localhost"
+        port = self.settings.get("PORT") or 27017
+        cmd += f" --host {host}:{port}"
+        if self.settings.get("USER"):
+            cmd += f" --username {self.settings['USER']}"
+        if self.settings.get("PASSWORD"):
+            cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}"
+
+        if self.settings.get("AUTH_SOURCE"):
+            cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}"
+        for collection in self.exclude:
+            cmd += f" --excludeCollection {collection}"
+        cmd += " --archive"
+        cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = self.restore_cmd
+        host = self.settings.get("HOST") or "localhost"
+        port = self.settings.get("PORT") or 27017
+        cmd += f" --host {host}:{port}"
+        if self.settings.get("USER"):
+            cmd += f" --username {self.settings['USER']}"
+        if self.settings.get("PASSWORD"):
+            cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}"
+
+        if self.settings.get("AUTH_SOURCE"):
+            cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}"
+        if self.object_check:
+            cmd += " --objcheck"
+        if self.drop:
+            cmd += " --drop"
+        cmd += " --archive"
+        cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
+        return self.run_command(cmd, stdin=dump, env=self.restore_env)

django_sync_env-0.2.0/src/django_sync_env/db/mysql.py

@@ -0,0 +1,45 @@
+from syncenv import utils
+
+from .base import BaseCommandDBConnector
+
+
+class MysqlDumpConnector(BaseCommandDBConnector):
+    """
+    MySQL connector, creates dump with ``mysqldump`` and restore with
+    ``mysql``.
+    """
+
+    dump_cmd = "mysqldump"
+    restore_cmd = "mysql"
+
+    def _create_dump(self):
+        cmd = f"{self.dump_cmd} {self.settings['NAME']} --quick"
+        if self.settings.get("HOST"):
+            cmd += f" --host={self.settings['HOST']}"
+        if self.settings.get("PORT"):
+            cmd += f" --port={self.settings['PORT']}"
+        if self.settings.get("USER"):
+            cmd += f" --user={self.settings['USER']}"
+        if self.settings.get("PASSWORD"):
+            cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}"
+
+        for table in self.exclude:
+            cmd += f" --ignore-table={self.settings['NAME']}.{table}"
+        cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = f"{self.restore_cmd} {self.settings['NAME']}"
+        if self.settings.get("HOST"):
+            cmd += f" --host={self.settings['HOST']}"
+        if self.settings.get("PORT"):
+            cmd += f" --port={self.settings['PORT']}"
+        if self.settings.get("USER"):
+            cmd += f" --user={self.settings['USER']}"
+        if self.settings.get("PASSWORD"):
+            cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}"
+
+        cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
+        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
+        return stdout, stderr

django_sync_env-0.2.0/src/django_sync_env/db/postgresql.py

@@ -0,0 +1,127 @@
+import logging
+from urllib.parse import quote
+
+from .base import BaseCommandDBConnector
+from .exceptions import DumpError
+
+logger = logging.getLogger("sync_env")
+
+
+def create_postgres_uri(self):
+    host = self.settings.get("HOST")
+    if not host:
+        raise DumpError("A host name is required")
+
+    dbname = self.settings.get("NAME") or ""
+    user = quote(self.settings.get("USER") or "")
+    password = self.settings.get("PASSWORD") or ""
+    password = f":{quote(password)}" if password else ""
+    if not user:
+        password = ""
+    else:
+        host = "@" + host
+
+    port = ":{}".format(self.settings.get("PORT")) if self.settings.get("PORT") else ""
+    dbname = f"--dbname=postgresql://{user}{password}{host}{port}/{dbname}"
+    return dbname
+
+
+class PgDumpConnector(BaseCommandDBConnector):
+    """
+    PostgreSQL connector, it uses pg_dump`` to create an SQL text file
+    and ``psql`` for restore it.
+    """
+
+    extension = "psql"
+    dump_cmd = "pg_dump"
+    restore_cmd = "psql"
+    single_transaction = True
+    drop = True
+
+    def _create_dump(self):
+        cmd = f"{self.dump_cmd} "
+        cmd = cmd + create_postgres_uri(self)
+
+        for table in self.exclude:
+            cmd += f" --exclude-table-data={table}"
+        if self.drop:
+            cmd += " --clean"
+
+        cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        cmd = f"{self.restore_cmd} "
+        cmd = cmd + create_postgres_uri(self)
+
+        # without this, psql terminates with an exit value of 0 regardless of errors
+        cmd += " --set ON_ERROR_STOP=on"
+        if self.single_transaction:
+            cmd += " --single-transaction"
+        cmd += " {}".format(self.settings["NAME"])
+        cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
+        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
+        return stdout, stderr
+
+
+class PgDumpGisConnector(PgDumpConnector):
+    """
+    PostgreGIS connector, same than :class:`PgDumpGisConnector` but enable
+    postgis if not made.
+    """
+
+    psql_cmd = "psql"
+
+    def _enable_postgis(self):
+        cmd = f'{self.psql_cmd} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'
+        cmd += " --username={}".format(self.settings["ADMIN_USER"])
+        cmd += " --no-password"
+        if self.settings.get("HOST"):
+            cmd += " --host={}".format(self.settings["HOST"])
+        if self.settings.get("PORT"):
+            cmd += " --port={}".format(self.settings["PORT"])
+        return self.run_command(cmd)
+
+    def _restore_dump(self, dump):
+        if self.settings.get("ADMIN_USER"):
+            self._enable_postgis()
+        return super()._restore_dump(dump)
+
+
+class PgDumpBinaryConnector(PgDumpConnector):
+    """
+    PostgreSQL connector, it uses pg_dump`` to create an SQL text file
+    and ``pg_restore`` for restore it.
+    """
+
+    extension = "psql.bin"
+    dump_cmd = "pg_dump"
+    restore_cmd = "pg_restore"
+    single_transaction = True
+    drop = True
+
+    def _create_dump(self):
+        cmd = f"{self.dump_cmd} "
+        cmd = cmd + create_postgres_uri(self)
+
+        cmd += " --format=custom"
+        for table in self.exclude:
+            cmd += f" --exclude-table-data={table}"
+        cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}"
+        logger.info(cmd)
+        stdout, stderr = self.run_command(cmd, env=self.dump_env)
+        return stdout
+
+    def _restore_dump(self, dump):
+        dbname = create_postgres_uri(self)
+        cmd = f"{self.restore_cmd} {dbname}"
+
+        if self.single_transaction:
+            cmd += " --single-transaction"
+        if self.drop:
+            cmd += " --clean"
+        cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}"
+        logger.info(cmd)
+        stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env)
+        return stdout, stderr