django-db-anonymiser 0.1.0__py3-none-any.whl

django_db_anonymiser/database_sanitizer/utils/mysql.py

@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import re
+
+import pymysql
+import six
+
+
+def get_mysqldump_args_and_env_from_url(url):
+    """
+    Constructs list of command line arguments and dictionary of environment
+    variables that can be given to `mysqldump` executable to obtain database
+    dump of the database described in given URL.
+
+    :param url: Parsed database URL.
+    :type url: urllib.urlparse.ParseResult
+
+    :return: List of command line arguments as well as dictionary of
+             environment variables that can be used to launch the MySQL dump
+             process to obtain dump of the database.
+    :rtype: tuple[list[str],dict[str,str]]
+    """
+    args = [
+        # Without this, `INSERT INTO` statements will exclude column names from
+        # the output, which are required for sanitation.
+        "--complete-insert",
+
+        # This enables use for "exteded inserts" where multiple rows of a table
+        # are included in a single `INSERT INTO` statement (contents of the
+        # entire table even, if it's within limits). We use it to increase the
+        # performance of the sanitation and to decrease the dump size.
+        "--extended-insert",
+
+        # This makes the `mysqldump` to attempt to limit size of a single line
+        # into 10 megabytes. We use it to reduce memory consumption.
+        "--net_buffer_length=10240",
+
+        # Hostname of the database to connect into, should be always present in
+        # the parsed database URL.
+        "-h",
+        url.hostname,
+    ]
+    env = {}
+
+    if url.port is not None:
+        args.extend(("-P", six.text_type(url.port)))
+
+    if url.username:
+        args.extend(("-u", url.username))
+
+    if url.password:
+        env["MYSQL_PWD"] = url.password
+
+    if len(url.path) < 2 or not url.path.startswith("/"):
+        raise ValueError("Name of the database is missing from the URL")
+
+    args.append(url.path[1:])
+
+    return args, env
+
+
+MYSQL_NULL_PATTERN = re.compile(r"^NULL$", re.IGNORECASE)
+MYSQL_BOOLEAN_PATTERN = re.compile(r"^(TRUE|FALSE)$", re.IGNORECASE)
+MYSQL_FLOAT_PATTERN = re.compile(r"^[+-]?\d*\.\d+([eE][+-]?\d+)?$")
+MYSQL_INT_PATTERN = re.compile(r"^\d+$")
+MYSQL_STRING_PATTERN = re.compile(r"'(?:[^']|''|\\')*(?<![\\])'")
+
+
+def decode_mysql_literal(text):
+    """
+    Attempts to decode given MySQL literal into Python value.
+
+    :param text: Value to be decoded, as MySQL literal.
+    :type text: str
+
+    :return: Python version of the given MySQL literal.
+    :rtype: any
+    """
+    if MYSQL_NULL_PATTERN.match(text):
+        return None
+
+    if MYSQL_BOOLEAN_PATTERN.match(text):
+        return text.lower() == "true"
+
+    if MYSQL_FLOAT_PATTERN.match(text):
+        return float(text)
+
+    if MYSQL_INT_PATTERN.match(text):
+        return int(text)
+
+    if MYSQL_STRING_PATTERN.match(text):
+        return decode_mysql_string_literal(text)
+
+    raise ValueError("Unable to decode given value: %r" % (text,))
+
+
+MYSQL_STRING_ESCAPE_SEQUENCE_PATTERN = re.compile(r"\\(.)")
+MYSQL_STRING_ESCAPE_SEQUENCE_MAPPING = {
+    "\\0": "\000",
+    "\\b": "\b",
+    "\\n": "\n",
+    "\\r": "\r",
+    "\\t": "\t",
+    "\\Z": "\032",
+}
+
+
+def decode_mysql_string_literal(text):
+    """
+    Removes quotes and decodes escape sequences from given MySQL string literal
+    returning the result.
+
+    :param text: MySQL string literal, with the quotes still included.
+    :type text: str
+
+    :return: Given string literal with quotes removed and escape sequences
+             decoded.
+    :rtype: str
+    """
+    assert text.startswith("'")
+    assert text.endswith("'")
+
+    # Ditch quotes from the string literal.
+    text = text[1:-1]
+
+    return MYSQL_STRING_ESCAPE_SEQUENCE_PATTERN.sub(
+        unescape_single_character,
+        text,
+    )
+
+
+def unescape_single_character(match):
+    """
+    Unescape a single escape sequence found from a MySQL string literal,
+    according to the rules defined at:
+    https://dev.mysql.com/doc/refman/5.6/en/string-literals.html#character-escape-sequences
+
+    :param match: Regular expression match object.
+
+    :return: Unescaped version of given escape sequence.
+    :rtype: str
+    """
+    value = match.group(0)
+    assert value.startswith("\\")
+    return MYSQL_STRING_ESCAPE_SEQUENCE_MAPPING.get(value) or value[1:]
+
+
+def encode_mysql_literal(value):
+    """
+    Converts given Python value into MySQL literal, suitable to be used inside
+    `INSERT INTO` statement.
+
+    :param value: Value to convert into MySQL literal.
+    :type value: any
+
+    :return: Given value encoded into MySQL literal.
+    :rtype: str
+    """
+    return pymysql.converters.escape_item(value, "utf-8")

django_db_anonymiser/database_sanitizer/utils/postgres.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+"""
+Contains utilities for working with Postgres `COPY` command, mainly encoding
+and decoding values in the custom format used by Postgres.
+
+Documentation about copy command and the text format used by it can be found
+from:
+https://www.postgresql.org/docs/9.2/static/sql-copy.html
+
+For decoding we use a regular expression to find the escape sequences
+and invoke `unescape_single_character` function for each occurence.
+Allowed escape sequences are precalculated into `DECODE_MAP` to make the
+lookups faster.
+
+For encoding we use a string translation table `ENCODE_TRANSLATE_TABLE`,
+which maps the "forbidden" characters to escape sequences.  This is used
+with `str.translate`, which is very fast way to escape characters.
+"""
+
+from __future__ import unicode_literals
+
+import itertools
+import re
+
+import six
+
+#: Representation of NULL value in Postgres COPY statement.
+POSTGRES_COPY_NULL_VALUE = "\\N"
+
+ENCODE_MAP = {
+    '\\': '\\\\',
+    '\b': '\\b',
+    '\f': '\\f',
+    '\n': '\\n',
+    '\r': '\\r',
+    '\t': '\\t',
+    '\v': '\\v',
+}
+
+ENCODE_TRANSLATE_TABLE = [
+    ENCODE_MAP.get(six.unichr(n), six.unichr(n))
+    for n in range(256)
+]
+
+DECODE_REGEX = re.compile(r"""
+\\                 # a backslash
+(?:                # followed by one of these (in non-capturing parenthesis):
+    [0-7]{1,3}         # 1, 2 or 3 octal digits
+    |                  # or
+    x[0-9a-fA-F]{1,2}  # 'x' followed by 1 or 2 hexadecimal digits
+    |                  # or
+    .                  # any character
+    |                  # or
+    \Z                 # end of string
+)
+""", re.VERBOSE)
+
+
+def decode_copy_value(value):
+    """
+    Decodes value received as part of Postgres `COPY` command.
+
+    :param value: Value to decode.
+    :type value: str
+
+    :return: Either None if the value is NULL string, or the given value where
+             escape sequences have been decoded from.
+    :rtype: str|None
+    """
+    # Test for null values first.
+    if value == POSTGRES_COPY_NULL_VALUE:
+        return None
+
+    # If there is no backslash present, there's nothing to decode.
+    #
+    # This early return provides a little speed-up, because it's very
+    # common to not have anything to decode and then simple search for
+    # backslash is faster than the regex sub below.
+    if '\\' not in value:
+        return value
+
+    return DECODE_REGEX.sub(unescape_single_character, value)
+
+
+def unescape_single_character(match):
+    """
+    Unescape a single escape sequence found by regular expression.
+
+    :param match: Regular expression match object
+    :rtype: str
+    :raises: ValueError if the escape sequence is invalid
+    """
+    try:
+        return DECODE_MAP[match.group(0)]
+    except KeyError:
+        value = match.group(0)
+        if value == '\\':
+            raise ValueError("Unterminated escape sequence encountered")
+
+        raise ValueError(
+            "Unrecognized escape sequence encountered: {}".format(value))
+
+
+def encode_copy_value(value):
+    """
+    Encodes given value into format suitable for Postgres `COPY` statement.
+
+    :param value: Value to encode.
+    :type value: str|None
+
+    :return: Given value encoded into format that is suitable to be used in the
+             `COPY` command.
+    :rtype: str
+    """
+    if value is None:
+        return POSTGRES_COPY_NULL_VALUE
+
+    return value.translate(ENCODE_TRANSLATE_TABLE)
+
+
+def _generate_decode_map():
+    # Initialize the map by inverting the encode map
+    decode_map = {
+        encoded_char: char
+        for (char, encoded_char) in ENCODE_MAP.items()
+    }
+
+    # Add entries for 1-3 octal digits and 1-2 hexadecimal digits
+    digit_encode_params = [
+        # (base, prefix, lengths, digit_chars)
+        (8, '\\', [1, 2, 3], '01234567'),
+        (16, '\\x', [1, 2], '0123456789abcdefABCDEF')
+    ]
+    for (base, prefix, lengths, digit_chars) in digit_encode_params:
+        for length in lengths:
+            for digits in itertools.product(digit_chars, repeat=length):
+                digit_string = ''.join(digits)
+                value = int(digit_string, base=base)
+                char = six.unichr(value)
+                decode_map[prefix + digit_string] = char
+
+    return decode_map
+
+
+DECODE_MAP = _generate_decode_map()

django_db_anonymiser/db_anonymiser/faker.py

@@ -0,0 +1,91 @@
+import os
+
+from faker import Faker
+
+fake = Faker("en-GB")
+
+
+def sanitize_name(value):
+    return fake.name()
+
+
+def sanitize_first_name(value):
+    return fake.first_name()
+
+
+def sanitize_last_name(value):
+    return fake.last_name()
+
+
+def sanitize_email(value):
+    return fake.unique.email()
+
+
+def sanitize_company_name(value):
+    return fake.unique.company()
+
+
+def sanitize_phone_number(value):
+    return "+44" + fake.msisdn()[3:]
+
+
+def sanitize_address(value):
+    return fake.address().replace("\n", ", ")
+
+
+def sanitize_website(value):
+    return fake.domain_name(2)
+
+
+def sanitize_text(value):
+    if not value:
+        return value
+    return fake.paragraph(nb_sentences=5)
+
+
+def sanitize_short_text(value):
+    if not value:
+        return value
+    return fake.text(max_nb_chars=100)
+
+
+def sanitize_street_address(value):
+    return fake.street_address()
+
+
+def sanitize_city(value):
+    return fake.city()
+
+
+def sanitize_postcode(value):
+    return fake.postcode()
+
+
+def sanitize_eori_number(value):
+    return "GB" + str(fake.random_number(digits=12))
+
+def sanitize_ni_eori_number(value):
+    return "XI" + str(fake.random_number(digits=12))
+
+def sanitize_eu_eori_number(value):
+    eu_countries = ["LU","BE", "FI", "FR", "CZ"]
+    return fake.random_element(eu_countries) + str(fake.random_number(digits=5))
+
+
+
+def sanitize_sic_number(value):
+    return str(fake.random_number(digits=5))
+
+
+def sanitize_vat_number(value):
+    return "GB" + str(fake.random_number(digits=9))
+
+
+def sanitize_registration_number(value):
+    return str(fake.random_number(digits=8))
+
+
+def sanitize_filename(value):
+    return os.path.split(
+        fake.file_path(extension=["pdf", "jpg", "csv", "txt", "docx", "ods", "xlsx"])
+    )[-1]

django_db_anonymiser/db_anonymiser/management/commands/dump_and_anonymise.py

@@ -0,0 +1,105 @@
+import os
+import logging
+
+import boto3
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+
+from django_db_anonymiser.database_sanitizer.config import Configuration
+from django_db_anonymiser.database_sanitizer.dump import run
+
+logger = logging.getLogger(__name__)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument(
+            "--keep-local-dumpfile",
+            action="store_true",
+            help="Keep local dump file, rather than cleaning it up.",
+        )
+        parser.add_argument(
+            "--skip-s3-upload",
+            action="store_true",
+            help="Skip uploading to S3.",
+        )
+
+    def configure(self):
+        self.keep_local_dumpfile = False
+        self.skip_s3_upload = False
+        self.dump_file_name = settings.DB_ANONYMISER_DUMP_FILE_NAME
+        self.temporary_dump_location = getattr(
+            settings,
+            "DB_ANONYMISER_TEMPORARY_DUMP_LOCATION",
+            f"/tmp/{self.dump_file_name}",
+        )
+        try:
+            self.config_location = settings.DB_ANONYMISER_CONFIG_LOCATION
+        except AttributeError:
+            raise CommandError(
+                "DB_ANONYMISER_CONFIG_LOCATION must be set in django settings."
+            )
+        additional_s3_params = {}
+        aws_endpoint_url = getattr(settings, "DB_ANONYMISER_AWS_ENDPOINT_URL", None)
+        if aws_endpoint_url:
+            additional_s3_params[
+                "endpoint_url"
+            ] = settings.DB_ANONYMISER_AWS_ENDPOINT_URL
+        self.s3_client = boto3.client(
+            "s3",
+            aws_access_key_id=settings.DB_ANONYMISER_AWS_ACCESS_KEY_ID,
+            aws_secret_access_key=settings.DB_ANONYMISER_AWS_SECRET_ACCESS_KEY,
+            region_name=settings.DB_ANONYMISER_AWS_REGION,
+            **additional_s3_params,
+        )
+        self.s3_bucket_name = settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
+
+    def handle(self, *args, **options):
+        logger.info("Starting DB dump and anonymiser")
+        self.configure()
+
+        if options["keep_local_dumpfile"]:
+            self.keep_local_dumpfile = True
+
+        if options["skip_s3_upload"]:
+            self.skip_s3_upload = True
+
+        try:
+            self.dump_anonymised_db()
+            self.write_to_s3()
+            logger.info("DB dump and anonymiser was successful!")
+        finally:
+            self.cleanup()
+
+    def dump_anonymised_db(self):
+        db_details = settings.DATABASES["default"]
+        postgres_url = f"postgresql://{db_details['USER']}:{db_details['PASSWORD']}@{db_details['HOST']}:{db_details['PORT']}/{db_details['NAME']}"
+        logger.info(
+            "Writing anonymised dumpfile to temporary location %s", self.dump_file_name
+        )
+        with open(self.temporary_dump_location, "w") as outfile:
+            run(
+                url=postgres_url,
+                config=Configuration.from_file(self.config_location),
+                output=outfile,
+            )
+        logger.info("Writing anonymised dumpfile complete")
+
+    def write_to_s3(self):
+        if self.skip_s3_upload:
+            return
+        logger.info("Writing file %s to S3", self.dump_file_name)
+        self.s3_client.upload_file(
+            self.temporary_dump_location, self.s3_bucket_name, self.dump_file_name
+        )
+        logger.info("Writing file to S3 complete")
+
+    def cleanup(self):
+        if self.keep_local_dumpfile:
+            return
+        logger.info("Cleaning up temporary files")
+        try:
+            os.remove(self.temporary_dump_location)
+        except FileNotFoundError:
+            pass
+        logger.info("Clean up complete")

django_db_anonymiser/db_anonymiser/tests/test_command.py

@@ -0,0 +1,90 @@
+import os
+from unittest.mock import patch
+
+from django.conf import settings
+from django.contrib.auth.models import User
+from django.core.management import call_command
+from django.test import TransactionTestCase
+
+import boto3
+import pytest
+from moto import mock_aws
+
+
+@pytest.mark.skipif(os.getenv("CIRCLECI") != "true", reason="Skipped because test requires real postgres db.")
+@mock_aws
+class TestDumpAndAnonmyiseCommand(TransactionTestCase):
+    def setUp(self):
+        self.aws = boto3.client("s3", region_name=settings.DB_ANONYMISER_AWS_REGION)
+        self.aws.create_bucket(
+            Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME,
+            CreateBucketConfiguration={
+                "LocationConstraint": settings.DB_ANONYMISER_AWS_REGION
+            },
+        )
+
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.run"
+    )
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.Configuration"
+    )
+    def test_dump_and_anonymise_calls_anonymiser(
+        self, mocked_configuration, mocked_anonymiser_run
+    ):
+        call_command(
+            "dump_and_anonymise", keep_local_dumpfile=False, skip_s3_upload=True
+        )
+        call_args, call_kwargs = mocked_anonymiser_run.call_args
+        assert (
+            call_kwargs["url"]
+            == f"postgresql://{settings.DATABASES['default']['USER']}:{settings.DATABASES['default']['PASSWORD']}@{settings.DATABASES['default']['HOST']}:{settings.DATABASES['default']['PORT']}/{settings.DATABASES['default']['NAME']}"
+        )
+        assert call_kwargs["config"] == mocked_configuration.from_file.return_value
+        assert call_kwargs["output"].name == "/tmp/anonymised.sql"
+        # Ensure skip_s3_upload was respected
+        bucket_contents = self.aws.list_objects(
+            Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
+        ).get("Contents", [])
+        assert bucket_contents == []
+
+    def test_dump_and_anonymise_writes_sql(self):
+        user = User.objects.create(
+            first_name="Bob",
+            last_name="Benson",
+            email="bob.benson@example.net",  # /PS-IGNORE
+            username="bob.benson",
+        )
+        call_command(
+            "dump_and_anonymise", keep_local_dumpfile=True, skip_s3_upload=True
+        )
+        dump_sql = ""
+        with open(f"/tmp/{settings.DB_ANONYMISER_DUMP_FILE_NAME}") as f:
+            dump_sql = f.read()
+        assert user.username in dump_sql
+        assert user.first_name not in dump_sql
+        assert user.last_name not in dump_sql
+        assert user.email not in dump_sql
+
+    def test_dump_and_anonymise_writes_to_s3(self):
+        call_command("dump_and_anonymise", keep_local_dumpfile=False)
+        bucket_contents = self.aws.list_objects(
+            Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
+        ).get("Contents", [])
+        assert bucket_contents[0]["Key"] == settings.DB_ANONYMISER_DUMP_FILE_NAME
+
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.os.remove"
+    )
+    def test_dump_and_anonymise_clears_local_file(self, mocked_os_remove):
+        call_command("dump_and_anonymise")
+        mocked_os_remove.assert_called_with(
+            f"/tmp/{settings.DB_ANONYMISER_DUMP_FILE_NAME}"
+        )
+
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.os.remove"
+    )
+    def test_dump_and_anonymise_keeps_local_file(self, mocked_os_remove):
+        call_command("dump_and_anonymise", keep_local_dumpfile=True)
+        assert not mocked_os_remove.called