django-dbdiff 0.9.5__tar.gz → 0.9.7__tar.gz

django_dbdiff-0.9.7/PKG-INFO

@@ -0,0 +1,151 @@
+Metadata-Version: 2.4
+Name: django-dbdiff
+Version: 0.9.7
+Summary: Database data diffing against fixtures for testing
+Author-email: James Pic <jamespic@gmail.com>
+License: MIT
+Project-URL: Homepage, https://github.com/yourlabs/django-dbdiff
+Keywords: django,test,database,fixture,diff
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Web Environment
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Framework :: Django
+Classifier: Framework :: Django :: 4.2
+Classifier: Framework :: Django :: 5.2
+Classifier: Framework :: Django :: 6.0
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Programming Language :: Python :: 3.14
+Classifier: Topic :: Internet :: WWW/HTTP
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.10
+Description-Content-Type: text/x-rst
+Requires-Dist: ijson
+Requires-Dist: json_delta
+
+.. image:: https://github.com/yourlabs/django-dbdiff/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/yourlabs/django-dbdiff/actions/workflows/ci.yml
+.. image:: https://codecov.io/gh/yourlabs/django-dbdiff/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/yourlabs/django-dbdiff
+.. image:: https://badge.fury.io/py/django-dbdiff.png
+   :target: http://badge.fury.io/py/django-dbdiff
+
+django-dbdiff
+~~~~~~~~~~~~~
+
+I'm pretty lazy when it comes to writing tests for existing code, however, I'm
+even lazier when it comes to repetitive manual testing action.
+
+This package aims at de-duplicating the data import tests from
+django-representatives and django-representatives-votes which is re-used in
+django-cities-light.
+
+Database state assertion
+========================
+
+A nice way to test a data import script is to create a source data fixture with
+a subset of data, ie. with only 10 cities instead of 28K or only 3 european
+parliament representatives instead of 3600, feed the import function with that
+and then compare the database state with a django fixture. This looks like what
+I was used to do:
+
+- use such a command to create a small data extract
+  `shuf -n3 cities15000.txt > cities_light/tests/cities_test_fixture.txt`,
+- use it against the import script on a clean database,
+- verify the database manually, and run
+  `django-admin dumpdata --indent=4 cities_light > cities_light/tests/cities_test_expected.txt`
+- then, make a test case that calls the import script against the fixture,
+- write and maintain some funny (fuzzy ?) repetitive test code to ensure that
+  the database is in the expected state.
+
+When a bug is fixed, just add the case to the fixture and repeat the process to
+create new expected data dumps, use coverage to ensure no case is missed.
+
+With django-dbdiff, I just need to maintain to initial data extract, and test
+it with ``Fixture('appname/path/to/fixture',
+models=[YourModelToTest]).assertNoDiff()`` in a
+``django.test.TransactionTestCase`` which has ``reset_sequences=True``:
+
+- if the fixture in question doesn't exist, it'll be automatically created on
+  with dumpdata for the concerned models on the first run, raising
+  "FixtureCreated" exception to fail the test and inform of the path of the
+  created fixture, so that it doesn't mislead the user in thinking the test
+  passed with an existing fixture,
+- if the fixture exists, it'll run dumpdata on the models concerned and GNU
+  diff it against the fixture, if there's any output it'll be raised in the
+  "DiffFound" exception, failing the test and printing the diff.
+
+Usage
+=====
+
+Example:
+
+.. code-block:: python
+
+    from django.test import TransactionTestCase
+    from dbdiff.fixture import Fixture
+
+
+    class YourImportTest(TransactionTestCase):
+        reset_sequences = True
+
+        def test_your_import(self):
+            your_import()
+
+            Fixture('yourapp/tests/yourtest.json',
+                    models=[YourModel]).assertNoDiff()
+
+The first time, it will raise a ``FixtureCreated`` exception, and the test will
+fail. This is to inform the user that the test didn't really run. On the next
+run though, it will pass.
+
+If any difference is found between the database and the test fixture, then
+``diff()`` will return the diff as outputed by GNU diff.
+
+If you need to ignore fields globally, set the class-level variable exclude as such:
+
+.. code-block:: python
+
+   Fixture.exclude = {'mrsrequest.mrsrequest': ['token']}
+
+If your import produces records with non-deterministic primary keys (e.g.
+UUIDs or sequence gaps), pass ``ignore_pk=True`` to match records by their
+field content instead of by pk:
+
+.. code-block:: python
+
+    Fixture('yourapp/tests/yourtest.json',
+            models=[YourModel],
+            ignore_pk=True).assertNoDiff()
+
+Instead of deleting the fixtures manually before running the tests to
+regenerate them, just run your tests with FIXTURE_REWRITE=1 environment
+variable. This will overwrite the fixtures and make the tests look like it
+passed.
+
+See tests and docstrings for crunchy details.
+
+Requirements
+============
+
+MySQL, SQLite and PostgreSQL, Python 3.10 to 3.14 are supported along with
+Django 4.2, 5.2, and 6.0 - it's always better to support django's master so
+that we can **upgrade easily when it is released**, which is one of the selling
+points for having 100% coverage.
+
+Install
+=======
+
+Install ``django-dbdiff`` with pip and add ``dbdiff`` to ``INSTALLED_APPS``.
+
+Django model observer
+=====================
+
+It is interresting to note that a related, perhaps sort-of similar app exists:
+https://github.com/Griffosx/djmo

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/README.rst

@@ -1,7 +1,7 @@
-.. image:: https://travis-ci.org/yourlabs/django-dbdiff.svg
-    :target: https://travis-ci.org/yourlabs/django-dbdiff
-.. image:: https://codecov.io/github/yourlabs/django-dbdiff/coverage.svg?branch=master
-    :target: https://codecov.io/github/yourlabs/django-dbdiff?branch=master
+.. image:: https://github.com/yourlabs/django-dbdiff/actions/workflows/ci.yml/badge.svg
+    :target: https://github.com/yourlabs/django-dbdiff/actions/workflows/ci.yml
+.. image:: https://codecov.io/gh/yourlabs/django-dbdiff/branch/master/graph/badge.svg
+    :target: https://codecov.io/gh/yourlabs/django-dbdiff
 .. image:: https://badge.fury.io/py/django-dbdiff.png
    :target: http://badge.fury.io/py/django-dbdiff
 
@@ -57,11 +57,11 @@ Example:
 
 .. code-block:: python
 
-    from django import TransactionTestCase
+    from django.test import TransactionTestCase
     from dbdiff.fixture import Fixture
 
 
-    class YourImportTest(test.TransactionTestCase):
+    class YourImportTest(TransactionTestCase):
         reset_sequences = True
 
         def test_your_import(self):
@@ -83,6 +83,16 @@ If you need to ignore fields globally, set the class-level variable exclude as s
 
    Fixture.exclude = {'mrsrequest.mrsrequest': ['token']}
 
+If your import produces records with non-deterministic primary keys (e.g.
+UUIDs or sequence gaps), pass ``ignore_pk=True`` to match records by their
+field content instead of by pk:
+
+.. code-block:: python
+
+    Fixture('yourapp/tests/yourtest.json',
+            models=[YourModel],
+            ignore_pk=True).assertNoDiff()
+
 Instead of deleting the fixtures manually before running the tests to
 regenerate them, just run your tests with FIXTURE_REWRITE=1 environment
 variable. This will overwrite the fixtures and make the tests look like it
@@ -93,10 +103,10 @@ See tests and docstrings for crunchy details.
 Requirements
 ============
 
-MySQL, SQLite and PostgreSQL, Python 2.7 and 3.4 are supported along with
-Django 1.7 to 1.10 - it's always better to support django's master so that we
-can **upgrade easily when it is released**, which is one of the selling points
-for having 100% coverage.
+MySQL, SQLite and PostgreSQL, Python 3.10 to 3.14 are supported along with
+Django 4.2, 5.2, and 6.0 - it's always better to support django's master so
+that we can **upgrade easily when it is released**, which is one of the selling
+points for having 100% coverage.
 
 Install
 =======

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/apps.py

@@ -9,8 +9,7 @@ from .utils import patch_transaction_test_case
 
 
 class DefaultConfig(AppConfig):
-    """
-    Register patched serializers and patch TransactionTestCase for sqlite.
+    """Register patched serializers and patch TransactionTestCase for sqlite.
 
     .. py:attribute:: debug
 
@@ -23,8 +22,7 @@ class DefaultConfig(AppConfig):
     default_indent = 4
 
     def ready(self):
-        """
-        Register dbdiff.serializers.json and set debug.
+        """Register dbdiff.serializers.json and set debug.
 
         Enables debug if a DBDIFF_DEBUG environment variable is found.
 

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/exceptions.py

@@ -59,8 +59,7 @@ class DiffFound(DbDiffException):
 
 
 class FixtureCreated(DbDiffException):
-    """
-    Raised when a fixture was created.
+    """Raised when a fixture was created.
 
     This purposely fails a test, to avoid misleading the user into thinking
     that the test was properly executed against a versioned fixture. Imagine

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/fixture.py

@@ -5,11 +5,10 @@ import json
 import os
 import tempfile
 
+import ijson
 from django.apps import apps
 from django.core.management import call_command
 
-import ijson
-
 from .exceptions import DiffFound, FixtureCreated
 from .utils import (
     diff,
@@ -18,13 +17,11 @@ from .utils import (
     get_tree,
 )
 
-
 REWRITE = os.getenv('FIXTURE_REWRITE')
 
 
 class Fixture(object):
-    """
-    Is able to print out diffs between database and a fixture.
+    """Is able to print out diffs between database and a fixture.
 
     .. py:attribute:: path
 
@@ -46,9 +43,10 @@ class Fixture(object):
 
     exclude = dict()
 
-    def __init__(self, relative_path, models=None, database=None):
-        """
-        Instanciate a FixtureDiff on a database.
+    def __init__(
+        self, relative_path, models=None, database=None, ignore_pk=False
+    ):
+        """Instanciate a FixtureDiff on a database.
 
         relative_path is used to calculate :py:attr:`path`, with
         :py:func:`~utils.get_absolute_path`.
@@ -58,14 +56,19 @@ class Fixture(object):
 
         database should be the name of the database to use, `default` by
         default.
+
+        ignore_pk when True, matches records by field content instead of
+        primary key. Records with the same content but different pks
+        are considered equal.
         """
         self.path = get_absolute_path(relative_path)
         self.models = models if models else self.parse_models()
         self.database = database or 'default'
+        self.ignore_pk = ignore_pk
 
     def parse_models(self):
         """Return the list of models inside the fixture file."""
-        with open(self.path, 'r') as f:
+        with open(self.path, 'rb') as f:
             return [apps.get_model(i.lower())
                     for i in ijson.items(f, 'item.model')]
 
@@ -91,31 +94,39 @@ class Fixture(object):
 
             return len(line) - len(line.lstrip(' '))
 
-    def diff(self, exclude=None):
-        """
-        Diff the fixture against a datadump of fixture models.
+    def diff(self, exclude=None, ignore_pk=None):
+        """Diff the fixture against a datadump of fixture models.
 
         If passed, exclude should be a list of field names to exclude from
         being diff'ed.
+
+        ignore_pk when True, matches records by field content instead of
+        primary key. Defaults to the instance's ignore_pk attribute.
         """
         fh, dump_path = tempfile.mkstemp('_dbdiff')
 
         exclude_final = copy.copy(self.exclude)
         exclude_final.update(exclude or {})
 
-        with os.fdopen(fh, 'w') as f:
-            self.dump(f)
+        try:
+            with os.fdopen(fh, 'w') as f:
+                self.dump(f)
 
-        with open(self.path, 'r') as e, open(dump_path, 'r') as r:
-            expected, result = json.load(e), json.load(r)
+            with open(self.path, 'r') as e, open(dump_path, 'r') as r:
+                expected, result = json.load(e), json.load(r)
 
-        unexpected, missing, different = diff(
-            get_tree(expected, exclude_final),
-            get_tree(result, exclude_final),
-        )
+            if ignore_pk is None:
+                ignore_pk = self.ignore_pk
 
-        if not unexpected and not missing and not diff:
+            unexpected, missing, different = diff(
+                get_tree(expected, exclude_final),
+                get_tree(result, exclude_final),
+                ignore_pk=ignore_pk,
+            )
+        finally:
             os.unlink(dump_path)
+
+        if not unexpected and not missing and not different:
             return None
 
         return unexpected, missing, different
@@ -150,8 +161,11 @@ class Fixture(object):
             if not REWRITE:
                 raise FixtureCreated(self)
 
-        unexpected, missing, different = self.diff(exclude=exclude)
+        result = self.diff(exclude=exclude)
+        if result is None:
+            return
 
+        unexpected, missing, different = result
         if unexpected or missing or different:
             raise DiffFound(self, unexpected, missing, different)
 

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/sequence.py

@@ -14,8 +14,7 @@ def pk_sequence_get(model):
 
 
 def sequence_reset(model):
-    """
-    Better sequence reset than TransactionTestCase.
+    """Better sequence reset than TransactionTestCase.
 
     The difference with using TransactionTestCase with reset_sequences=True is
     that this will reset sequences for the given models to their higher value,
@@ -49,7 +48,7 @@ def sequence_reset(model):
                 column=pk_field, table=table
             )
         )
-        result = cursor.fetchone()[0] or 0
+        result = cursor.fetchone()[0] or 1
         reset = 'ALTER TABLE {table} AUTO_INCREMENT = %s' % result
 
     connection.cursor().execute(

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/serializers/base.py

@@ -10,8 +10,7 @@ class BaseSerializerMixin(object):
 
     @classmethod
     def recursive_dict_sort(cls, data):
-        """
-        Return a recursive OrderedDict for a dict.
+        """Return a recursive OrderedDict for a dict.
 
         Django's default model-to-dict logic - implemented in
         django.core.serializers.python.Serializer.get_dump_object() - returns a
@@ -28,8 +27,7 @@ class BaseSerializerMixin(object):
 
     @classmethod
     def remove_microseconds(cls, data):
-        """
-        Strip microseconds from datetimes for mysql.
+        """Strip microseconds from datetimes for mysql.
 
         MySQL doesn't have microseconds in datetimes, so dbdiff's serializer
         removes microseconds from datetimes so that fixtures are cross-database
@@ -51,8 +49,7 @@ class BaseSerializerMixin(object):
 
     @classmethod
     def normalize_decimals(cls, data):
-        """
-        Strip trailing zeros for constitency.
+        """Strip trailing zeros for consistency.
 
         In addition, dbdiff serialization forces Decimal normalization, because
         trailing zeros could happen in inconsistent ways.
@@ -67,8 +64,7 @@ class BaseSerializerMixin(object):
                 data['fields'][key] = value.normalize()
 
     def get_dump_object(self, obj):
-        """
-        Actual method used by Django serializers to dump dicts.
+        """Actual method used by Django serializers to dump dicts.
 
         By overridding this method, we're able to run our various
         data dump predictability methods.

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/serializers/json.py

@@ -4,7 +4,6 @@ from django.core.serializers import json as upstream
 
 from .base import BaseSerializerMixin
 
-
 __all__ = ('Serializer', 'Deserializer')
 
 

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/test.py

@@ -6,8 +6,7 @@ from .sequence import sequence_reset
 
 
 class DbdiffTestMixin(object):
-    """
-    Convenience mixin with better sequence resetting than TransactionTestCase.
+    """Mixin with better sequence resetting than TransactionTestCase.
 
     The difference with using TransactionTestCase with reset_sequences=True is
     that this will reset sequences for the given models to their higher value,
@@ -52,6 +51,7 @@ class DbdiffTestMixin(object):
         Fixture(
             self.dbdiff_expected,
             models=self.dbdiff_models,
+            ignore_pk=getattr(self, 'dbdiff_ignore_pk', False),
         ).assertNoDiff(
             exclude=self.dbdiff_exclude,
         )

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/decimal_test/migrations/0001_initial.py

@@ -1,7 +1,4 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/decimal_test/migrations/0002_auto_20160102_0914.py

@@ -1,7 +1,4 @@
-# -*- coding: utf-8 -*-
-from __future__ import unicode_literals
-
-from django.db import models, migrations
+from django.db import migrations, models
 
 
 class Migration(migrations.Migration):

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/project/settings.py

@@ -1,5 +1,4 @@
-"""
-Django settings for project project.
+"""Django settings for project project.
 
 Generated by 'django-admin startproject' using Django 1.8.3.dev20150604012123.
 

django_dbdiff-0.9.7/dbdiff/tests/project/settings_mysql.py

@@ -0,0 +1,21 @@
+from .settings import *  # noqa
+
+import os
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.mysql',
+        'HOST': os.environ.get('DB_HOST', ''),
+        'NAME': os.environ.get('DB_NAME', 'dbdiff_test'),
+        'USER': os.environ.get('DB_USER', 'root'),
+        'PASSWORD': os.environ.get('DB_PASSWORD', ''),
+        'PORT': os.environ.get('DB_PORT', ''),
+        'OPTIONS': {
+            'charset': 'utf8mb4',
+        },
+        'TEST':{
+            'CHARSET': 'utf8mb4',
+            'COLLATION': 'utf8mb4_unicode_ci',
+        }
+    }
+}

django_dbdiff-0.9.7/dbdiff/tests/project/settings_postgresql.py

@@ -0,0 +1,16 @@
+from .settings import *  # noqa
+
+import os
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'HOST': os.environ.get('DB_HOST', ''),
+        'NAME': os.environ.get('DB_NAME', 'dbdiff_test'),
+        'USER': os.environ.get('DB_USER', 'postgres'),
+        'PASSWORD': os.environ.get('DB_PASSWORD', ''),
+        'PORT': os.environ.get('DB_PORT', '5432'),
+        'OPTIONS': {},
+
+    }
+}

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/test_compare.py

@@ -1,15 +1,11 @@
 """Public API tests."""
 
-from __future__ import unicode_literals
-
 import os
 import tempfile
 
 from django import test
 from django.contrib.auth.models import Group
 
-import six
-
 from ..exceptions import DiffFound, FixtureCreated
 from ..fixture import Fixture
 
@@ -57,8 +53,8 @@ class SmokeTest(test.TransactionTestCase):
 1 instance(s) of auth.group have not expected fields
 #1:
   name:
-- u'testgroup'
-+ u'BOOM'
+- 'testgroup'
++ 'BOOM'
 '''
 
         with self.assertRaises(DiffFound) as result:
@@ -73,12 +69,12 @@ class SmokeTest(test.TransactionTestCase):
         expected = '''
 1 unexpected instance(s) of auth.group found in the dump:
 #2:
-{u'name': u'unexpected', u'permissions': []}
+{'name': 'unexpected', 'permissions': []}
 1 instance(s) of auth.group have not expected fields
 #1:
   name:
-- u'testgroup'
-+ u'BOOM'
+- 'testgroup'
++ 'BOOM'
 '''
 
         with self.assertRaises(DiffFound) as result:
@@ -90,10 +86,10 @@ class SmokeTest(test.TransactionTestCase):
         expected = '''
 1 unexpected instance(s) of auth.group found in the dump:
 #2:
-{u'name': u'unexpected', u'permissions': []}
+{'name': 'unexpected', 'permissions': []}
 1 expected instance(s) of auth.group missing from dump:
 #1:
-{u'name': u'testgroup', u'permissions': []}
+{'name': 'testgroup', 'permissions': []}
 '''
 
         with self.assertRaises(DiffFound) as result:
@@ -101,13 +97,6 @@ class SmokeTest(test.TransactionTestCase):
         self.assert_message_is(expected, result)
 
     def assert_message_is(self, expected, result):
-        if six.PY3:
-            expected = expected.replace("u'", "'")
-
-        msg = (
-            result.exception.message
-            if six.PY2 else result.exception.args[0]
-        )
-
+        msg = result.exception.args[0]
         out = '\n'.join(msg.split('\n')[1:])
         assert out.strip() == expected.strip()

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/test_fixture.py

@@ -23,3 +23,12 @@ class FixtureTest(test.TransactionTestCase):
 
     def test_models(self):
         assert self.fixture.models == [Group]
+
+    def test_diff_exact_match_returns_none(self):
+        # Regression: the always-False `not diff` (imported function) bug
+        # caused diff() to never return None, leaking temp files and breaking
+        # assertNoDiff() with a TypeError on exact matches.
+        # Fixture: # [{"model": "auth.group", "pk": 1,
+        # "fields": {"name": "initial_name"}}]
+        Group.objects.create(id=1, name='initial_name')
+        assert self.fixture.diff() is None

{django-dbdiff-0.9.5 → django_dbdiff-0.9.7}/dbdiff/tests/test_mixin.py

@@ -1,14 +1,15 @@
-from dbdiff.test import DbdiffTestMixin
-
 from django import test
 from django.contrib.contenttypes.models import ContentType
 from django.db import connection
 
+from dbdiff.test import DbdiffTestMixin
+
 
 class ContentTypeTestCase(DbdiffTestMixin, test.TestCase):
     dbdiff_models = [ContentType]
     dbdiff_exclude = {'*': ['created']}
     dbdiff_reset_sequences = True
+    dbdiff_ignore_pk = True
     dbdiff_expected = 'dbdiff/tests/test_mixin.json'
 
     def test_db_import(self):