django-dbdiff 0.9.6__py3-none-any.whl

dbdiff/__init__.py

@@ -0,0 +1,3 @@
+"""dbdiff module enable diffing a fixture against the database."""
+
+default_app_config = 'dbdiff.apps.DefaultConfig'

dbdiff/apps.py

@@ -0,0 +1,44 @@
+"""AppConfig for dbdiff."""
+
+import os
+
+from django.apps import AppConfig
+from django.core.serializers import register_serializer
+
+from .utils import patch_transaction_test_case
+
+
+class DefaultConfig(AppConfig):
+    """
+    Register patched serializers and patch TransactionTestCase for sqlite.
+
+    .. py:attribute:: debug
+
+        If True, then diff commands will be printed to stdout and temporary
+        files will not be deleted.
+    """
+
+    name = 'dbdiff'
+    debug = False
+    default_indent = 4
+
+    def ready(self):
+        """
+        Register dbdiff.serializers.json and set debug.
+
+        Enables debug if a DBDIFF_DEBUG environment variable is found.
+
+        It is important to use serializers which dump data in a predictible way
+        because this app relies on diff between an expected - user-generated
+        and versioned - fixture and dumped database data. This method also
+        overrides the default json serializer with dbdiff's.
+
+        When dbdiff is installed, ``dumpdata`` will use its serializers which
+        have predictible output and cross-database support, so fixtures dumped
+        without dbdiff installed will have to be regenerated after dbdiff is
+        installed to be usable with dbdiff.
+
+        """
+        self.debug = os.environ.get('DBDIFF_DEBUG', False)
+        register_serializer('json', 'dbdiff.serializers.json')
+        patch_transaction_test_case()

dbdiff/exceptions.py

@@ -0,0 +1,72 @@
+"""Exceptions for dbdiff module."""
+import pprint
+
+
+class DbDiffException(Exception):
+    """Base exception for this app."""
+
+
+class DiffFound(DbDiffException):
+    """Raised when a diff is found by the context manager."""
+
+    def _add_messages(self, msg, title, tree):
+        if tree:
+            for model, instances in tree.items():
+                msg.append(
+                    title % (
+                        len(instances),
+                        model
+                    )
+                )
+
+                for pk, fields in instances.items():
+                    msg.append('#%s:\n%s' % (pk, pprint.pformat(fields)))
+
+    def __init__(self, fixture, unexpected, missing, diff):
+        """Exception for when a diff was found."""
+        msg = ['Diff found with dump at %s' % fixture.path]
+
+        self._add_messages(
+            msg,
+            '%s unexpected instance(s) of %s found in the dump:',
+            unexpected
+        )
+
+        self._add_messages(
+            msg,
+            '%s expected instance(s) of %s missing from dump:',
+            missing
+        )
+
+        if diff:
+            for model, instances in diff.items():
+                msg.append(
+                    '%s instance(s) of %s have not expected fields' % (
+                        len(instances),
+                        model
+                    )
+                )
+
+                for pk, fields in instances.items():
+                    msg.append('#%s:' % pk)
+
+                    for field, values in fields.items():
+                        msg.append('  %s:' % field)
+                        msg.append('- %s' % pprint.pformat(values[0]))
+                        msg.append('+ %s' % pprint.pformat(values[1]))
+
+        super(DiffFound, self).__init__('\n'.join(msg))
+
+
+class FixtureCreated(DbDiffException):
+    """
+    Raised when a fixture was created.
+
+    This purposely fails a test, to avoid misleading the user into thinking
+    that the test was properly executed against a versioned fixture. Imagine
+    one pushes a test without the fixture, it will break because of this
+    exception in CI.
+
+    However, this should only happen once per fixture - unless the user in
+    question forgets to commit the generated fixture !
+    """

dbdiff/fixture.py

@@ -0,0 +1,172 @@
+"""Public fixture API."""
+
+import copy
+import json
+import os
+import tempfile
+
+from django.apps import apps
+from django.core.management import call_command
+
+import ijson
+
+from .exceptions import DiffFound, FixtureCreated
+from .utils import (
+    diff,
+    get_absolute_path,
+    get_model_names,
+    get_tree,
+)
+
+
+REWRITE = os.getenv('FIXTURE_REWRITE')
+
+
+class Fixture(object):
+    """
+    Is able to print out diffs between database and a fixture.
+
+    .. py:attribute:: path
+
+        Absolute path to the fixture.
+
+    .. py:attribute:: models
+
+        List of models concerned by the fixture.
+
+    .. py:attribute:: database
+
+        Database name to use, 'default' by default.
+
+    .. py:attribute:: exclude
+
+        Class attribute, dict of model fields to ignore in the form of:
+        {'app.model': ['fieldN']}
+    """
+
+    exclude = dict()
+
+    def __init__(self, relative_path, models=None, database=None, ignore_pk=False):
+        """
+        Instanciate a FixtureDiff on a database.
+
+        relative_path is used to calculate :py:attr:`path`, with
+        :py:func:`~utils.get_absolute_path`.
+
+        If models is None, then it will be generated from reading the
+        fixture file, but generated fixtures will include all models.
+
+        database should be the name of the database to use, `default` by
+        default.
+
+        ignore_pk when True, matches records by field content instead of
+        primary key. Records with the same content but different pks
+        are considered equal.
+        """
+        self.path = get_absolute_path(relative_path)
+        self.models = models if models else self.parse_models()
+        self.database = database or 'default'
+        self.ignore_pk = ignore_pk
+
+    def parse_models(self):
+        """Return the list of models inside the fixture file."""
+        with open(self.path, 'r') as f:
+            return [apps.get_model(i.lower())
+                    for i in ijson.items(f, 'item.model')]
+
+    @property
+    def exists(self):
+        """Return True if :py:attr:`path` exists."""
+        return os.path.exists(self.path)
+
+    @property
+    def indent(self):
+        """Return the indentation of the fixture file or the default indent."""
+        if not os.path.exists(self.path):
+            return apps.get_app_config('dbdiff').default_indent
+
+        with open(self.path, 'r') as f:
+            line = f.readline()
+
+            while line and ':' not in line:
+                line = f.readline()
+
+            if not line:
+                return apps.get_app_config('dbdiff').default_indent
+
+            return len(line) - len(line.lstrip(' '))
+
+    def diff(self, exclude=None, ignore_pk=None):
+        """
+        Diff the fixture against a datadump of fixture models.
+
+        If passed, exclude should be a list of field names to exclude from
+        being diff'ed.
+
+        ignore_pk when True, matches records by field content instead of
+        primary key. Defaults to the instance's ignore_pk attribute.
+        """
+        fh, dump_path = tempfile.mkstemp('_dbdiff')
+
+        exclude_final = copy.copy(self.exclude)
+        exclude_final.update(exclude or {})
+
+        with os.fdopen(fh, 'w') as f:
+            self.dump(f)
+
+        with open(self.path, 'r') as e, open(dump_path, 'r') as r:
+            expected, result = json.load(e), json.load(r)
+
+        if ignore_pk is None:
+            ignore_pk = self.ignore_pk
+
+        unexpected, missing, different = diff(
+            get_tree(expected, exclude_final),
+            get_tree(result, exclude_final),
+            ignore_pk=ignore_pk,
+        )
+
+        if not unexpected and not missing and not diff:
+            os.unlink(dump_path)
+            return None
+
+        return unexpected, missing, different
+
+    def load(self):
+        """Load fixture into the database."""
+        call_command('loaddata', self.path)
+
+    def dump(self, out):
+        """Write fixture with dumpdata for fixture models."""
+        call_command(
+            'dumpdata',
+            *get_model_names(self.models),
+            format='json',
+            traceback=True,
+            indent=self.indent,
+            stdout=out,
+            use_natural_foreign_keys=True
+        )
+
+    def assertNoDiff(self, exclude=None):  # noqa
+        """Assert that the fixture doesn't have any diff with the database.
+
+        If the fixture doesn't exist then it's written but
+        :py:class:`~exceptions.FixtureCreated` is raised.
+
+        If a diff was found it will raise :py:class:`~exceptions.DiffFound`.
+        """
+        if REWRITE or not self.exists:
+            with open(self.path, 'w+') as f:
+                self.dump(f)
+            if not REWRITE:
+                raise FixtureCreated(self)
+
+        unexpected, missing, different = self.diff(exclude=exclude)
+
+        if unexpected or missing or different:
+            raise DiffFound(self, unexpected, missing, different)
+
+    def __str__(self):
+        """Return :py:attr:`path` for string representation."""
+        return self.path

dbdiff/plugin.py

@@ -0,0 +1,35 @@
+"""Pytest plugin for django-dbdiff.
+
+The marker enables the smarter sequence reset feature previously available in
+the DbdiffTestMixin in pytest, example usage::
+
+    @dbdiff(models=[YourModel])
+    def your_test():
+        assert YourModel.objects.create().pk == 1
+"""
+import pytest
+
+from .sequence import sequence_reset
+
+
+@pytest.fixture(autouse=True)
+def _dbdiff_marker(request):
+    marker = request.node.get_closest_marker('dbdiff')
+    if not marker:
+        return
+
+    # Enable transactional db
+    request.getfixturevalue('transactional_db')
+
+    for model in marker.kwargs['models']:
+        sequence_reset(model)
+
+
+def pytest_load_initial_conftests(early_config, parser, args):
+    """Register the dbdiff mark."""
+    early_config.addinivalue_line(
+        'markers',
+        'dbdiff(models, reset_sequences=True): Mark the test as using '
+        'the django test database.  The *transaction* argument marks will '
+        "allow you to use real transactions in the test like Django's "
+        'TransactionTestCase.')

dbdiff/sequence.py

@@ -0,0 +1,57 @@
+"""Smarter model pk sequence reset."""
+from django.db import connection, models
+
+
+def pk_sequence_get(model):
+    """Return a list of table, column tuples which should have sequences."""
+    for field in model._meta.get_fields():
+        if not getattr(field, 'primary_key', False):
+            continue
+        if not isinstance(field, models.AutoField):
+            continue
+        return (field.db_column or field.column, field.model._meta.db_table)
+    return (None, None)
+
+
+def sequence_reset(model):
+    """
+    Better sequence reset than TransactionTestCase.
+
+    The difference with using TransactionTestCase with reset_sequences=True is
+    that this will reset sequences for the given models to their higher value,
+    supporting pre-existing models which could have been created by a
+    migration.
+    """
+    pk_field, table = pk_sequence_get(model)
+    if not pk_field:
+        return
+
+    if connection.vendor == 'postgresql':
+        reset = """
+        SELECT
+            setval(
+                pg_get_serial_sequence('{table}', '{column}'),
+                coalesce(max({column}),0) + 1,
+                false
+            )
+        FROM {table}
+        """
+    elif connection.vendor == 'sqlite':
+        reset = """
+        UPDATE sqlite_sequence
+        SET seq=(SELECT max({column}) from {table})
+        WHERE name='{table}'
+        """
+    elif connection.vendor == 'mysql':
+        cursor = connection.cursor()
+        cursor.execute(
+            'SELECT MAX({column}) + 1 FROM {table}'.format(
+                column=pk_field, table=table
+            )
+        )
+        result = cursor.fetchone()[0] or 0
+        reset = 'ALTER TABLE {table} AUTO_INCREMENT = %s' % result
+
+    connection.cursor().execute(
+        reset.format(column=pk_field, table=table)
+    )

dbdiff/serializers/__init__.py

@@ -0,0 +1 @@
+"""Serializers with predictible (ordered) output."""

dbdiff/serializers/base.py

@@ -0,0 +1,80 @@
+"""Shared code for serializers."""
+
+import collections
+import datetime
+import decimal
+
+
+class BaseSerializerMixin(object):
+    """Serializer mixin for predictible and cross-db dumps."""
+
+    @classmethod
+    def recursive_dict_sort(cls, data):
+        """
+        Return a recursive OrderedDict for a dict.
+
+        Django's default model-to-dict logic - implemented in
+        django.core.serializers.python.Serializer.get_dump_object() - returns a
+        dict, this app registers a slightly modified version of the default
+        json serializer which returns OrderedDicts instead.
+        """
+        ordered_data = collections.OrderedDict(sorted(data.items()))
+
+        for key, value in ordered_data.items():
+            if isinstance(value, dict):
+                ordered_data[key] = cls.recursive_dict_sort(value)
+
+        return ordered_data
+
+    @classmethod
+    def remove_microseconds(cls, data):
+        """
+        Strip microseconds from datetimes for mysql.
+
+        MySQL doesn't have microseconds in datetimes, so dbdiff's serializer
+        removes microseconds from datetimes so that fixtures are cross-database
+        compatible which make them usable for cross-database testing.
+        """
+        for key, value in data['fields'].items():
+            if not isinstance(value, datetime.datetime):
+                continue
+
+            data['fields'][key] = datetime.datetime(
+                year=value.year,
+                month=value.month,
+                day=value.day,
+                hour=value.hour,
+                minute=value.minute,
+                second=value.second,
+                tzinfo=value.tzinfo
+            )
+
+    @classmethod
+    def normalize_decimals(cls, data):
+        """
+        Strip trailing zeros for constitency.
+
+        In addition, dbdiff serialization forces Decimal normalization, because
+        trailing zeros could happen in inconsistent ways.
+        """
+        for key, value in data['fields'].items():
+            if not isinstance(value, decimal.Decimal):
+                continue
+
+            if value % 1 == 0:
+                data['fields'][key] = int(value)
+            else:
+                data['fields'][key] = value.normalize()
+
+    def get_dump_object(self, obj):
+        """
+        Actual method used by Django serializers to dump dicts.
+
+        By overridding this method, we're able to run our various
+        data dump predictability methods.
+        """
+        data = super(BaseSerializerMixin, self).get_dump_object(obj)
+        self.remove_microseconds(data)
+        self.normalize_decimals(data)
+        data = self.recursive_dict_sort(data)
+        return data

dbdiff/serializers/json.py

@@ -0,0 +1,15 @@
+"""Django JSON Serializer override."""
+
+from django.core.serializers import json as upstream
+
+from .base import BaseSerializerMixin
+
+
+__all__ = ('Serializer', 'Deserializer')
+
+
+class Serializer(BaseSerializerMixin, upstream.Serializer):
+    """Sorted dict JSON serializer."""
+
+
+Deserializer = upstream.Deserializer

dbdiff/test.py

@@ -0,0 +1,58 @@
+"""Convenience test mixin."""
+from django.core.management import call_command
+
+from .fixture import Fixture
+from .sequence import sequence_reset
+
+
+class DbdiffTestMixin(object):
+    """
+    Convenience mixin with better sequence resetting than TransactionTestCase.
+
+    The difference with using TransactionTestCase with reset_sequences=True is
+    that this will reset sequences for the given models to their higher value,
+    supporting pre-existing models which could have been created by a
+    migration.
+
+    The test case subclass requires some attributes and an implementation of a
+    ``dbdiff_test()`` method that does the actual import call that this
+    should test. Example usage::
+
+        class FrancedataImportTest(DbdiffTestMixin, test.TestCase):
+            dbdiff_models = [YourModel]
+            dbdiff_exclude = {'*': ['created']}
+            dbdiff_reset_sequences = True
+            dbdiff_expected = 'yourapp/tests/yourexpectedfixture.json'
+            dbdiff_fixtures = ['your-fixtures.json']
+
+            def dbdiff_test(self):
+                fixture = os.path.join(
+                    os.path.dirname(__file__),
+                    'representatives_fixture.json'
+                )
+
+                with open(fixture, 'r') as f:
+                    do_your_import.main(f)
+
+    Supports postgresql.
+    """
+
+    def test_db_import(self):
+        """Actual test method, ran by the test suite."""
+        call_command('flush', interactive=False)
+
+        for fixture in getattr(self, 'dbdiff_fixtures', []):
+            call_command('loaddata', fixture)
+
+        for model in self.dbdiff_models:
+            sequence_reset(model)
+
+        self.dbdiff_test()
+
+        Fixture(
+            self.dbdiff_expected,
+            models=self.dbdiff_models,
+            ignore_pk=getattr(self, 'dbdiff_ignore_pk', False),
+        ).assertNoDiff(
+            exclude=self.dbdiff_exclude,
+        )

dbdiff/tests/__init__.py

@@ -0,0 +1 @@
+"""Tests for the dbdiff module."""

dbdiff/tests/decimal_test/__init__.py

@@ -0,0 +1 @@
+"""App to test DecimalField dump predictibility."""

dbdiff/tests/decimal_test/migrations/0001_initial.py

@@ -0,0 +1,17 @@
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='TestModel',
+            fields=[
+                ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)),
+                ('test_field', models.DecimalField(max_digits=3, decimal_places=3)),
+            ],
+        ),
+    ]

dbdiff/tests/decimal_test/migrations/0002_auto_20160102_0914.py

@@ -0,0 +1,16 @@
+from django.db import models, migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('decimal_test', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='testmodel',
+            name='test_field',
+            field=models.DecimalField(max_digits=5, decimal_places=2),
+        ),
+    ]

dbdiff/tests/decimal_test/models.py

@@ -0,0 +1,5 @@
+from django.db import models
+
+
+class TestModel(models.Model):
+    test_field = models.DecimalField(max_digits=5, decimal_places=2)

dbdiff/tests/inheritance/__init__.py

@@ -0,0 +1 @@
+"""Test app for model inheritance."""

dbdiff/tests/inheritance/models.py

@@ -0,0 +1,9 @@
+from django.db import models
+
+
+class Parent(models.Model):
+    pass
+
+
+class Child(Parent):
+    name = models.CharField(max_length=50)

dbdiff/tests/nonintpk/__init__.py

@@ -0,0 +1 @@
+"""Test that we don't crash with non sequence pks."""

dbdiff/tests/nonintpk/models.py

@@ -0,0 +1,8 @@
+import uuid
+
+from django.db import models
+
+
+class Nonintpk(models.Model):
+    # dbdiff should not try to reset this sequence
+    id = models.UUIDField(primary_key=True, default=uuid.uuid4, editable=False)

dbdiff/tests/project/__init__.py

@@ -0,0 +1 @@
+"""Test project settings."""