django-db-anonymiser 0.1.0__py3-none-any.whl

django_db_anonymiser/database_sanitizer/tests/test_config.py

@@ -0,0 +1,256 @@
+# -*- coding: utf-8 -*-
+
+from collections import namedtuple
+
+import mock
+import pytest
+
+from .. import config
+from ..config import Configuration, ConfigurationError
+
+
+@mock.patch.object(config, 'open')
+@mock.patch('yaml.safe_load')
+def test_from_file(mocked_yaml_load, mocked_open):
+    mocked_yaml_load.return_value = {}
+
+    Configuration.from_file('filename.yml')
+
+    assert mocked_open.call_args == (('filename.yml', 'rb'), {})
+    opened_file = mocked_open.return_value.__enter__.return_value
+    assert mocked_yaml_load.call_args == ((opened_file,), {})
+
+
+def test_load_config_data_must_be_dict():
+    config = Configuration()
+    config.load({})
+    with pytest.raises(ConfigurationError):
+        config.load(config_data="test")
+
+
+def test_load_dump_extra_parameters():
+    config = Configuration()
+
+    config.load_dump_extra_parameters({})
+    assert config.mysqldump_params == ["--single-transaction"]
+    assert config.pg_dump_params == []
+
+    with pytest.raises(ConfigurationError):
+        config.load_dump_extra_parameters({"config": "test"})
+
+    config.load_dump_extra_parameters({"config": {}})
+    assert config.mysqldump_params == ["--single-transaction"]
+    assert config.pg_dump_params == []
+
+    with pytest.raises(ConfigurationError):
+        config.load_dump_extra_parameters({"config": {
+            "extra_parameters": "test"
+        }})
+
+    with pytest.raises(ConfigurationError):
+        config.load_dump_extra_parameters({"config": {
+            "extra_parameters": [True]
+        }})
+
+    with pytest.raises(ConfigurationError):
+        config.load_dump_extra_parameters({"config": {
+            "extra_parameters": {
+                "mysqldump": "hernekeitto",
+            },
+        }})
+
+    with pytest.raises(ConfigurationError):
+        config.load_dump_extra_parameters({"config": {
+            "extra_parameters": {
+                "pg_dump": "viina",
+            },
+        }})
+
+    config.load_dump_extra_parameters({"config": {
+        "extra_parameters": {
+            "mysqldump": ["--double-transaction"],
+            "pg_dump": ["--exclude-table=something"],
+        },
+    }})
+    assert config.mysqldump_params == ["--double-transaction"]
+    assert config.pg_dump_params == ["--exclude-table=something"]
+
+
+def test_load_addon_packages():
+    config = Configuration()
+
+    config.load_addon_packages({})
+    assert config.addon_packages == []
+
+    with pytest.raises(ConfigurationError):
+        config.load_addon_packages({"config": "test"})
+
+    config.load_addon_packages({"config": {}})
+    assert config.addon_packages == []
+
+    with pytest.raises(ConfigurationError):
+        config.load_addon_packages({"config": {"addons": "test"}})
+
+    with pytest.raises(ConfigurationError):
+        config.load_addon_packages({"config": {"addons": [True]}})
+
+    config.load_addon_packages({"config": {
+        "addons": [
+            "test1",
+            "test2",
+            "test3",
+        ],
+    }})
+    assert config.addon_packages == ["test1", "test2", "test3"]
+
+
+def test_load_sanitizers():
+    config = Configuration()
+
+    with pytest.raises(ConfigurationError):
+        config.load_sanitizers({"strategy": "test"})
+
+    with pytest.raises(ConfigurationError):
+        config.load_sanitizers({"strategy": {"test": "test"}})
+
+    def mock_find_sanitizer(*args):
+        return lambda value: value
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer",
+                    side_effect=mock_find_sanitizer):
+        with pytest.raises(ConfigurationError):
+            config.load_sanitizers({"strategy": {"table1": {"column1": True}}})
+
+        config.load_sanitizers({"strategy": {
+            "table1": {
+                "column1": None,
+                "column2": "test.test",
+            },
+            "table2": {
+                "column1": "test.test",
+            },
+            "table3": None,
+        }})
+
+    assert "table1.column1" not in config.sanitizers
+    assert "table1.column2" in config.sanitizers
+    assert "table2.column1" in config.sanitizers
+
+
+def test_table_skip_rows_configuration():
+    config = Configuration()
+
+    with pytest.raises(ConfigurationError):
+        config.load_sanitizers({"strategy": "test"})
+
+    def mock_find_sanitizer(*args):
+        return lambda value: value
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer",
+                    side_effect=mock_find_sanitizer):
+
+        config.load_sanitizers({"strategy": {
+            "table1": "skip_rows",
+            "table2": {
+                "column1": "test",
+            }
+        }})
+
+    assert "table2.column1" in config.sanitizers
+    assert "table1" in config.skip_rows_for_tables
+
+
+def test_find_sanitizer():
+    config = Configuration()
+
+    with pytest.raises(ConfigurationError):
+        config.find_sanitizer("test")
+
+    def mock_find_sanitizer_from_module1(module_name, function_name):
+        assert module_name == "sanitizers.test"
+        assert function_name == "sanitize_test"
+        return lambda value: value
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer_from_module",
+                    side_effect=mock_find_sanitizer_from_module1):
+        assert config.find_sanitizer("test.test") is not None
+
+    def mock_find_sanitizer_from_module2(module_name, function_name):
+        assert module_name in ("sanitizers.test", "addon.test")
+        assert function_name == "sanitize_test"
+        if module_name.startswith("addon."):
+            return lambda value: value
+        else:
+            return None
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer_from_module",
+                    side_effect=mock_find_sanitizer_from_module2):
+        config.addon_packages = ("addon",)
+        assert config.find_sanitizer("test.test") is not None
+
+    def mock_find_sanitizer_from_module3(module_name, function_name):
+        assert module_name in (
+            "sanitizers.test",
+            "addon.test",
+            "database_sanitizer.sanitizers.test",
+        )
+        assert function_name == "sanitize_test"
+        if module_name.startswith("database_sanitizer."):
+            return lambda value: value
+        else:
+            return None
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer_from_module",
+                    side_effect=mock_find_sanitizer_from_module3):
+        assert config.find_sanitizer("test.test") is not None
+
+    def mock_find_sanitizer_from_module4(module_name, function_name):
+        return None
+
+    with mock.patch("database_sanitizer.config.Configuration.find_sanitizer_from_module",
+                    side_effect=mock_find_sanitizer_from_module4):
+        with pytest.raises(ConfigurationError):
+            config.find_sanitizer("test.test")
+
+
+def test_find_sanitizer_from_module():
+    def mock_import1(module_name):
+        assert module_name == "test"
+        raise ImportError("Should be catched")
+
+    with mock.patch("importlib.import_module", side_effect=mock_import1):
+        assert Configuration.find_sanitizer_from_module("test", "test") is None
+
+    mock_module_type = namedtuple("mock_module", ("test",))
+
+    def mock_import2(module_name):
+        assert module_name == "test"
+        return mock_module_type(test=None)
+
+    with mock.patch("importlib.import_module", side_effect=mock_import2):
+        assert Configuration.find_sanitizer_from_module("test", "test") is None
+
+    def mock_import3(module_name):
+        assert module_name == "test"
+        return mock_module_type(test=lambda value: value)
+
+    with mock.patch("importlib.import_module", side_effect=mock_import3):
+        assert Configuration.find_sanitizer_from_module("test", "test") is not None
+
+    def mock_import4(module_name):
+        assert module_name == "test"
+        return mock_module_type(test="test")
+
+    with mock.patch("importlib.import_module", side_effect=mock_import4):
+        with pytest.raises(ConfigurationError):
+            Configuration.find_sanitizer_from_module("test", "test")
+
+
+def test_sanitize():
+    config = Configuration()
+    config.sanitizers["a.a"] = lambda value: value.upper()
+    config.sanitizers["a.b"] = lambda value: value[::-1]
+
+    assert config.sanitize("a", "a", "test") == "TEST"
+    assert config.sanitize("a", "b", "test") == "tset"
+    assert config.sanitize("a", "c", "test") == "test"

django_db_anonymiser/database_sanitizer/tests/test_dump.py

@@ -0,0 +1,123 @@
+import subprocess
+from io import BytesIO, StringIO
+
+import mock
+import pytest
+
+from database_sanitizer import dump
+from database_sanitizer.config import Configuration
+
+EXPECTED_POPEN_KWARGS = {
+    'mysql://User:Pass@HostName/Db': {
+        'args': (
+            'mysqldump --complete-insert --extended-insert'
+            ' --net_buffer_length=10240 -h hostname -u User Db'
+            ' --single-transaction'
+        ).split(),
+        'env': {'MYSQL_PWD': 'Pass'},
+        'stdout': subprocess.PIPE,
+    },
+    'postgres:///Db': {
+        'args': tuple((
+            'pg_dump --encoding=utf-8 --quote-all-identifiers'
+            ' --dbname postgres:///Db').split()),
+        'stdout': subprocess.PIPE,
+    },
+}
+
+for url in ['postgresql:///Db', 'postgis:///Db']:
+    EXPECTED_POPEN_KWARGS[url] = EXPECTED_POPEN_KWARGS['postgres:///Db'].copy()
+    EXPECTED_POPEN_KWARGS[url]['args'] = tuple(
+        ' '.join(EXPECTED_POPEN_KWARGS[url]['args'])
+        .replace('postgres', 'postgresql').split())
+
+
+@pytest.mark.parametrize('url', list(EXPECTED_POPEN_KWARGS))
+@mock.patch('subprocess.Popen')
+def test_run(mocked_popen, url):
+    mocked_popen.return_value.stdout = BytesIO(b'INPUT DUMP')
+    output = StringIO()
+    config = None
+    dump.run(url, output, config)
+
+    expected_popen_kwargs = EXPECTED_POPEN_KWARGS[url]
+    (popen_args, popen_kwargs) = mocked_popen.call_args
+    expected_popen_args = (
+        (expected_popen_kwargs.pop('args'),) if popen_args else ())
+    assert popen_args == expected_popen_args
+    assert popen_kwargs == expected_popen_kwargs
+
+
+@mock.patch('subprocess.Popen')
+def test_run_with_mysql_extra_params(mocked_popen):
+    mocked_popen.return_value.stdout = BytesIO(b'INPUT DUMP')
+    output = StringIO()
+
+    url = "mysql://User:Pass@HostName/Db"
+    config = Configuration()
+    config.load({
+        "config": {
+            "extra_parameters": {
+                "mysqldump": ["--double-transaction"]
+            }
+        }
+    })
+
+    dump.run(url, output, config)
+
+    expected = {
+        'args': (
+            'mysqldump --complete-insert --extended-insert'
+            ' --net_buffer_length=10240 -h hostname -u User Db'
+            ' --double-transaction'
+        ).split(),
+        'env': {'MYSQL_PWD': 'Pass'},
+        'stdout': subprocess.PIPE,
+    }
+
+    (popen_args, popen_kwargs) = mocked_popen.call_args
+    expected_popen_args = (
+        (expected.pop('args'),) if popen_args else ())
+    assert popen_args == expected_popen_args
+    assert popen_kwargs == expected
+
+
+@mock.patch('subprocess.Popen')
+def test_run_with_pg_dump_extra_params(mocked_popen):
+    mocked_popen.return_value.stdout = BytesIO(b'INPUT DUMP')
+    output = StringIO()
+
+    url = "postgres:///Db"
+    config = Configuration()
+    config.load({
+        "config": {
+            "extra_parameters": {
+                "pg_dump": ["--exclude-table=something"]
+            }
+        }
+    })
+
+    dump.run(url, output, config)
+
+    expected = {
+        'args': tuple((
+            'pg_dump --encoding=utf-8 --quote-all-identifiers'
+            ' --dbname postgres:///Db'
+            ' --exclude-table=something'
+        ).split()),
+        'stdout': subprocess.PIPE,
+    }
+
+    (popen_args, popen_kwargs) = mocked_popen.call_args
+    expected_popen_args = (
+        (expected.pop('args'),) if popen_args else ())
+    assert popen_args == expected_popen_args
+    assert popen_kwargs == expected
+
+
+@mock.patch('subprocess.Popen')
+def test_run_unknown_scheme(mocked_popen):
+    with pytest.raises(ValueError) as excinfo:
+        dump.run('unknown:///db', None, None)
+    assert str(excinfo.value) == "Unsupported database scheme: 'unknown'"
+    mocked_popen.assert_not_called()

django_db_anonymiser/database_sanitizer/tests/test_dump_mysql.py

@@ -0,0 +1,196 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import io
+
+import pytest
+from six.moves.urllib import parse as urlparse
+
+from ..config import Configuration
+from ..dump.mysql import (
+    parse_column_names,
+    parse_values,
+    sanitize,
+    sanitize_from_stream,
+)
+
+MOCK_MYSQLDUMP_OUTPUT = b"""
+--- Fake MySQL database dump
+
+DROP TABLE IF EXISTS `test`;
+
+CREATE TABLE `test` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`created_at` date NOT NULL,
+`notes` varchar(255) NOT NULL,
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+INSERT INTO `test` (`id`, `created_at`, `notes`) VALUES \
+(1,'2018-01-01','Test data 1'),\
+(2,'2018-01-02','Test data 2'),\
+(3,'2018-01-03','Test data 3');
+
+--- Final line after `INSERT INTO` statement.
+"""
+
+MOCK_MYSQLDUMP_OUTPUT_WITH_U2028 = b"""
+--- Fake MySQL database dump
+
+DROP TABLE IF EXISTS `test`;
+
+CREATE TABLE `test` (
+`id` int(11) NOT NULL AUTO_INCREMENT,
+`created_at` date NOT NULL,
+`notes` varchar(255) NOT NULL,
+PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+INSERT INTO `test` (`id`, `created_at`, `notes`) VALUES \
+(1,'2018-01-01','Test \xe2\x80\xa8 data 1'),\
+(2,'2018-01-02','Test data 2'),\
+(3,'2018-01-03','Test data 3');
+
+--- Final line after `INSERT INTO` statement.
+"""
+
+
+INVALID_MOCK_MYSQLDUMP_OUTPUT = b"""
+--- Fake MySQL database dump
+
+DROP TABLE IF EXISTS `test`;
+
+INSERT INTO `test` (`id`, `created_at`, `notes`) VALUES (1),(2),(3);
+
+--- Final line after `INSERT INTO` statement.
+"""
+
+
+def test_sanitize_wrong_scheme():
+    url = urlparse.urlparse("http://localhost/test")
+    with pytest.raises(ValueError):
+        list(sanitize(url, None))
+
+
+def test_sanitize_from_stream():
+    stream = io.BytesIO(MOCK_MYSQLDUMP_OUTPUT)
+    config = Configuration()
+    config.sanitizers["test.notes"] = lambda value: "Sanitized"
+    dump_output_lines = list(sanitize_from_stream(stream, config))
+
+    assert "--- Fake MySQL database dump" in dump_output_lines
+    assert "--- Final line after `INSERT INTO` statement." in dump_output_lines
+    assert """INSERT INTO `test` (`id`, `created_at`, `notes`) VALUES \
+(1,'2018-01-01','Sanitized'),\
+(2,'2018-01-02','Sanitized'),\
+(3,'2018-01-03','Sanitized');\
+""" in dump_output_lines
+
+def test_sanitize_with_u2028_from_stream():
+    stream = io.BytesIO(MOCK_MYSQLDUMP_OUTPUT_WITH_U2028)
+    config = Configuration()
+    config.sanitizers["test.notes"] = lambda value: "Sanitized"
+    dump_output_lines = list(sanitize_from_stream(stream, config))
+
+    assert "--- Fake MySQL database dump" in dump_output_lines
+    assert "--- Final line after `INSERT INTO` statement." in dump_output_lines
+    assert """INSERT INTO `test` (`id`, `created_at`, `notes`) VALUES \
+(1,'2018-01-01','Sanitized'),\
+(2,'2018-01-02','Sanitized'),\
+(3,'2018-01-03','Sanitized');\
+""" in dump_output_lines
+
+
+def test_skip_table_rows():
+    stream = io.BytesIO(MOCK_MYSQLDUMP_OUTPUT)
+    config = Configuration()
+    config.skip_rows_for_tables.append('test')
+
+    output = list(sanitize_from_stream(stream, config))
+
+    assert output == [
+        '',
+        '--- Fake MySQL database dump',
+        '',
+        'DROP TABLE IF EXISTS `test`;',
+        '',
+        'CREATE TABLE `test` (',
+        '`id` int(11) NOT NULL AUTO_INCREMENT,',
+        '`created_at` date NOT NULL,',
+        '`notes` varchar(255) NOT NULL,',
+        'PRIMARY KEY (`id`)',
+        ') ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;',
+        '',
+        '',
+        '--- Final line after `INSERT INTO` statement.',
+    ]
+
+
+def test_sanitizer_invalid_input():
+    stream = io.BytesIO(INVALID_MOCK_MYSQLDUMP_OUTPUT)
+    config = Configuration()
+    config.sanitizers["test.notes"] = lambda value: "Sanitized"
+
+    with pytest.raises(ValueError):
+        list(sanitize_from_stream(stream, config))
+
+
+@pytest.mark.parametrize(
+    "text,expected_column_names",
+    (
+        ("`test`", ("test",)),
+        ("`test`, `test`", ("test", "test")),
+        ("`test`,`test`", ("test", "test")),
+    ),
+)
+def test_parse_column_names(text, expected_column_names):
+    assert parse_column_names(text) == expected_column_names
+
+
+@pytest.mark.parametrize(
+    "text,expected_values",
+    (
+        ("('test'),('test')", (("test",), ("test",))),
+        ("(1,2),(3,4),", ((1, 2), (3, 4))),
+        ("(TRUE),(FALSE),(NULL)", ((True,), (False,), (None,))),
+        ("(x')", ()),  # Invalid data
+    ),
+)
+def test_parse_values(text, expected_values):
+    assert tuple(parse_values(text)) == expected_values
+
+
+@pytest.mark.parametrize('config_type', [
+    'no-config', 'empty-config', 'single-column-config'])
+@pytest.mark.parametrize('data_label', ['ok', 'invalid'])
+def test_optimizations(config_type, data_label):
+    if config_type == 'no-config':
+        config = None
+        decoder_call_count = 0
+    else:
+        config = Configuration()
+        if config_type == 'empty-config':
+            decoder_call_count = 0
+        else:
+            assert config_type == 'single-column-config'
+            config.sanitizers["test.notes"] = (lambda x: x)
+            decoder_call_count = 3  # Number of rows in test table
+
+    data = {
+        'ok': MOCK_MYSQLDUMP_OUTPUT,
+        'invalid': INVALID_MOCK_MYSQLDUMP_OUTPUT,
+    }[data_label]
+
+    should_raise = (
+        config_type == 'single-column-config'
+        and data_label == 'invalid')
+
+    dump_stream = io.BytesIO(data)
+    if should_raise:
+        with pytest.raises(ValueError):
+            list(sanitize_from_stream(dump_stream, config))
+    else:
+        expected_output = data.decode('utf-8').splitlines()
+        result = list(sanitize_from_stream(dump_stream, config))
+        assert result == expected_output