django-db-anonymiser 0.1.0__py3-none-any.whl

django_db_anonymiser/database_sanitizer/dump/postgres.py

@@ -0,0 +1,170 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import codecs
+import io
+import re
+import subprocess
+
+from ..utils.postgres import decode_copy_value, encode_copy_value
+from ..config import PG_DUMP_DEFAULT_PARAMETERS
+
+COPY_LINE_PATTERN = re.compile(
+    r"^COPY \"(?P<schema>[^\"]*)\".\"(?P<table>[^\"]*)\" "
+    r"\((?P<columns>.*)\) "
+    r"FROM stdin;$"
+)
+
+
+def sanitize(url, config):
+    """
+    Obtains dump of an Postgres database by executing `pg_dump` command and
+    sanitizes it's output.
+
+    :param url: URL to the database which is going to be sanitized, parsed by
+                Python's URL parser.
+    :type url: six.moves.urllib.parse.ParseResult
+
+    :param config: Optional sanitizer configuration to be used for sanitation
+                   of the values stored in the database.
+    :type config: database_sanitizer.config.Configuration|None
+    """
+    if url.scheme not in ("postgres", "postgresql", "postgis"):
+        raise ValueError("Unsupported database type: '%s'" % (url.scheme,))
+
+    extra_params = PG_DUMP_DEFAULT_PARAMETERS
+    if config:
+        extra_params = config.pg_dump_params
+
+    process = subprocess.Popen(
+        (
+            "pg_dump",
+            # Force output to be UTF-8 encoded.
+            "--encoding=utf-8",
+            # Quote all table and column names, just in case.
+            "--quote-all-identifiers",
+            # Luckily `pg_dump` supports DB URLs, so we can just pass it the
+            # URL as argument to the command.
+            "--dbname",
+            url.geturl().replace('postgis://', 'postgresql://'),
+         ) + tuple(extra_params),
+        stdout=subprocess.PIPE,
+    )
+
+    sanitize_value_line = None
+    current_table = None
+    current_table_columns = None
+    skip_table = False
+
+    for line in io.TextIOWrapper(process.stdout, encoding="utf-8"):
+        # Eat the trailing new line.
+        line = line.rstrip("\n")
+
+        # Are we currently in middle of `COPY` statement?
+        if current_table:
+            # Backslash following a dot marks end of an `COPY` statement.
+            if line == "\\.":
+                current_table = None
+                current_table_columns = None
+                if not skip_table:
+                    yield "\\."
+                skip_table = False
+                continue
+
+            if skip_table:
+                continue
+
+            if not sanitize_value_line:
+                yield line
+                continue
+
+            yield sanitize_value_line(line)
+            continue
+
+        # Is the line beginning of `COPY` statement?
+        copy_line_match = COPY_LINE_PATTERN.match(line)
+        if not copy_line_match:
+            yield line
+            continue
+
+        current_table = copy_line_match.group("table")
+        current_table_columns = parse_column_names(copy_line_match.group("columns"))
+
+        # Skip `COPY` statement if table rows are configured
+        # to be skipped.
+        if config and current_table in config.skip_rows_for_tables:
+            skip_table = True
+            continue
+
+        sanitize_value_line = get_value_line_sanitizer(
+            config, current_table, current_table_columns)
+
+        yield line
+
+
+def get_value_line_sanitizer(config, table, columns):
+    if not config:
+        return None
+
+    def get_sanitizer(column):
+        sanitizer = config.get_sanitizer_for(table, column)
+
+        if not sanitizer:
+            return _identity
+
+        def decode_sanitize_encode(value):
+            return encode_copy_value(sanitizer(decode_copy_value(value)))
+
+        return decode_sanitize_encode
+
+    sanitizers = [get_sanitizer(column) for column in columns]
+
+    if all(x is _identity for x in sanitizers):
+        return None
+
+    def sanitize_line(line):
+        values = line.split('\t')
+        if len(values) != len(columns):
+            raise ValueError("Mismatch between column names and values.")
+        return '\t'.join(
+            sanitizer(value)
+            for (sanitizer, value) in zip(sanitizers, values))
+
+    return sanitize_line
+
+
+def _identity(x):
+    return x
+
+
+def parse_column_names(text):
+    """
+    Extracts column names from a string containing quoted and comma separated
+    column names.
+
+    :param text: Line extracted from `COPY` statement containing quoted and
+                 comma separated column names.
+    :type text: str
+
+    :return: Tuple containing just the column names.
+    :rtype: tuple[str]
+    """
+    return tuple(
+        re.sub(r"^\"(.*)\"$", r"\1", column_name.strip())
+        for column_name in text.split(",")
+    )
+
+
+def parse_values(text):
+    """
+    Parses line following `COPY` statement containing values for a single row
+    in the table, in custom Postgres format.
+
+    :param text: Line following `COPY` statement containing values.
+    :type text: str
+
+    :return: Column values extracted from the given line.
+    :rtype: tuple[str|None]
+    """
+    return tuple(decode_copy_value(value) for value in text.split("\t"))

django_db_anonymiser/database_sanitizer/sanitizers/constant.py

@@ -0,0 +1,14 @@
+def sanitize_null(value):
+    return None
+
+
+def sanitize_empty_json_dict(value):
+    return '{}'
+
+
+def sanitize_empty_json_list(value):
+    return '[]'
+
+
+def sanitize_invalid_django_password(value):
+    return '!'

django_db_anonymiser/database_sanitizer/sanitizers/derived.py

@@ -0,0 +1,14 @@
+import uuid
+
+from database_sanitizer.session import hash_text
+
+NIL_UUID = '00000000-0000-0000-0000-000000000000'
+NIL_UUID_WITHOUT_DASHES = NIL_UUID.replace('-', '')
+
+
+def sanitize_uuid4(value):
+    if not value:
+        return value
+    if value.replace('-', '') == NIL_UUID_WITHOUT_DASHES:
+        return NIL_UUID
+    return str(uuid.UUID(hash_text(value)[:32], version=4))

django_db_anonymiser/database_sanitizer/sanitizers/string.py

@@ -0,0 +1,31 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, unicode_literals
+
+import random
+import string
+
+CHARACTERS = string.ascii_letters + string.digits
+
+
+def sanitize_empty(value):
+    """
+    Built-in sanitizer which replaces the original value with empty string.
+    """
+    return None if value is None else ""
+
+
+def sanitize_zfill(value):
+    """
+    Built-in sanitizer which replaces the original value with zeros.
+    """
+    return None if value is None else "".zfill(len(value))
+
+
+def sanitize_random(value):
+    """
+    Random string of same length as the given value.
+    """
+    if not value:
+        return value
+    return ''.join(random.choice(CHARACTERS) for _ in range(len(value)))

django_db_anonymiser/database_sanitizer/sanitizers/times.py

@@ -0,0 +1,11 @@
+import datetime
+import random
+
+TEN_YEARS_AS_SECONDS = 10 * 365 * 24 * 3600
+
+
+def sanitize_random_past_timestamp(value):
+    num = random.randint(0, TEN_YEARS_AS_SECONDS * 1000)
+    delta = datetime.timedelta(seconds=(num / 1000.0))
+    dt = datetime.datetime.now() - delta
+    return dt.isoformat()

django_db_anonymiser/database_sanitizer/sanitizers/user.py

@@ -0,0 +1,145 @@
+from __future__ import unicode_literals
+
+from six import text_type
+
+from database_sanitizer.session import hash_text_to_int, hash_text_to_ints
+
+
+def sanitize_email(value):
+    if not value:
+        return value
+    (num1, num2, num3) = hash_text_to_ints(value.strip(), [16, 16, 32])
+    given_name = given_names[num1 % given_names_count]
+    surname = surnames[num2 % surnames_count]
+    case_convert = (text_type.lower if num3 % 8 > 0 else lambda x: x)
+    return '{first}.{last}@x{num:x}.sanitized.net'.format(
+        first=case_convert(given_name),
+        last=case_convert(surname).replace("'", ''),
+        num=num3)
+
+
+def sanitize_username(value):
+    if not value:
+        return value
+    (num1, num2) = hash_text_to_ints(value, [16, 32])
+    return '{}{:x}'.format(given_names[num1 % given_names_count].lower(), num2)
+
+
+def sanitize_full_name_en_gb(value):
+    if not value:
+        return value
+    (num1, num2) = hash_text_to_ints(value.strip().lower(), [16, 16])
+    return '{} {}'.format(
+        given_names[num1 % given_names_count], surnames[num2 % surnames_count])
+
+
+def sanitize_given_name_en_gb(value):
+    if not value:
+        return value
+    num = hash_text_to_int(value.strip().lower())
+    return given_names[num % given_names_count]
+
+
+def sanitize_surname_en_gb(value):
+    if not value:
+        return value
+    num = hash_text_to_int(value.strip().lower())
+    return surnames[num % surnames_count]
+
+
+given_names = """
+Aaron Abbie Abdul Abigail Adam Adrian Aimee Alan Albert Alex
+Alexander Alexandra Alice Alison Allan Amanda Amber Amelia Amy Andrea
+Andrew Angela Ann Anna Anne Annette Anthony Antony Arthur Ashleigh
+Ashley Barbara Barry Ben Benjamin Bernard Beth Bethan Bethany Beverley
+Billy Bradley Brandon Brenda Brett Brian Bruce Bryan Callum Cameron Carl
+Carly Carol Carole Caroline Carolyn Catherine Charlene Charles Charlie
+Charlotte Chelsea Cheryl Chloe Christian Christine Christopher Claire
+Clare Clifford Clive Colin Connor Conor Craig Dale Damian Damien Daniel
+Danielle Danny Darren David Dawn Dean Deborah Debra Declan Denis Denise
+Dennis Derek Diana Diane Dominic Donald Donna Dorothy Douglas Duncan
+Dylan Edward Eileen Elaine Eleanor Elizabeth Ellie Elliot Elliott Emily
+Emma Eric Fiona Frances Francesca Francis Frank Frederick Gail Gareth
+Garry Gary Gavin Gemma Geoffrey George Georgia Georgina Gerald Geraldine
+Gerard Gillian Glen Glenn Gordon Grace Graeme Graham Gregory Guy Hannah
+Harriet Harry Hayley Hazel Heather Helen Henry Hilary Hollie Holly
+Howard Hugh Iain Ian Irene Jack Jacob Jacqueline Jade Jake James Jamie
+Jane Janet Janice Jasmine Jason Jay Jayne Jean Jeffrey Jemma Jenna
+Jennifer Jeremy Jessica Jill Joan Joanna Joanne Jodie Joe Joel John
+Jonathan Jordan Joseph Josephine Josh Joshua Joyce Judith Julia Julian
+Julie June Justin Karen Karl Kate Katherine Kathleen Kathryn Katie Katy
+Kayleigh Keith Kelly Kenneth Kerry Kevin Kieran Kim Kimberley Kirsty
+Kyle Laura Lauren Lawrence Leah Leanne Lee Leigh Leon Leonard Lesley
+Leslie Lewis Liam Linda Lindsey Lisa Lorraine Louis Louise Lucy Luke
+Lydia Lynda Lynn Lynne Malcolm Mandy Marc Marcus Margaret Maria Marian
+Marie Marilyn Marion Mark Martin Martyn Mary Mathew Matthew Maureen
+Maurice Max Megan Melanie Melissa Michael Michelle Mitchell Mohamed
+Mohammad Mohammed Molly Naomi Natalie Natasha Nathan Neil Nicholas
+Nicola Nicole Nigel Norman Oliver Olivia Owen Paige Pamela Patricia
+Patrick Paul Paula Pauline Peter Philip Phillip Rachael Rachel Raymond
+Rebecca Reece Rhys Richard Ricky Rita Robert Robin Roger Ronald Rosemary
+Rosie Ross Roy Russell Ruth Ryan Sally Sam Samantha Samuel Sandra Sara
+Sarah Scott Sean Shane Shannon Sharon Shaun Sheila Shirley Sian Simon
+Sophie Stacey Stanley Stephanie Stephen Steven Stewart Stuart Susan
+Suzanne Sylvia Terence Teresa Terry Thomas Timothy Tina Toby Tom Tony
+Tracey Tracy Trevor Valerie Vanessa Victor Victoria Vincent Wayne Wendy
+William Yvonne Zoe
+""".strip().split()
+
+
+surnames = """
+Abbott Adams Ahmed Akhtar Alexander Ali Allan Allen Anderson Andrews
+Archer Armstrong Arnold Ashton Atkins Atkinson Austin Bailey Baker
+Baldwin Ball Banks Barber Barker Barlow Barnes Barnett Barrett Barry
+Bartlett Barton Bates Baxter Begum Bell Bennett Benson Bentley Berry
+Bevan Bibi Birch Bird Bishop Black Blackburn Bolton Bond Booth Bowen
+Boyle Bradley Bradshaw Brady Bray Brennan Briggs Brookes Brooks Brown
+Browne Bruce Bryan Bryant Bull Burgess Burke Burns Burrows Burton
+Butcher Butler Byrne Cameron Campbell Carey Carpenter Carr Carroll
+Carter Cartwright Chadwick Chambers Chan Chandler Chapman Charlton Clark
+Clarke Clayton Clements Coates Cole Coleman Coles Collier Collins
+Connolly Connor Conway Cook Cooke Cooper Cox Craig Crawford Cross
+Cunningham Curtis Dale Daly Daniels Davey Davidson Davies Davis Davison
+Dawson Day Dean Dennis Dickinson Dixon Dobson Dodd Doherty Donnelly
+Douglas Doyle Duffy Duncan Dunn Dyer Edwards Elliott Ellis Evans Farmer
+Farrell Faulkner Ferguson Field Finch Fisher Fitzgerald Fleming Fletcher
+Flynn Ford Forster Foster Fowler Fox Francis Franklin Fraser Freeman
+French Frost Fry Fuller Gallagher Gardiner Gardner Garner George Gibbons
+Gibbs Gibson Gilbert Giles Gill Glover Goddard Godfrey Goodwin Gordon
+Gough Gould Graham Grant Gray Green Greenwood Gregory Griffin Griffiths
+Hale Hall Hamilton Hammond Hancock Hanson Harding Hardy Hargreaves
+Harper Harris Harrison Hart Hartley Harvey Hawkins Hayes Haynes Hayward
+Heath Henderson Henry Herbert Hewitt Hicks Higgins Hill Hilton Hodgson
+Holden Holland Holloway Holmes Holt Hooper Hope Hopkins Horton Houghton
+Howard Howarth Howe Howell Howells Hudson Hughes Humphreys Humphries
+Hunt Hunter Hurst Hussain Hutchinson Hyde Ingram Iqbal Jackson James
+Jarvis Jenkins Jennings John Johnson Johnston Jones Jordan Joyce Kaur
+Kay Kelly Kemp Kennedy Kent Kerr Khan King Kirby Kirk Knight Knowles
+Lamb Lambert Lane Law Lawrence Lawson Leach Lee Lees Leonard Lewis
+Little Lloyd Long Lord Lowe Lucas Lynch Lyons Macdonald Mahmood Mann
+Manning Marsden Marsh Marshall Martin Mason Matthews May McCarthy
+McDonald McKenzie McLean Mellor Metcalfe Miah Middleton Miles Miller
+Mills Mistry Mitchell Moore Moran Morgan Morley Morris Morrison Morton
+Moss Murphy Murray Myers Nash Naylor Nelson Newman Newton Nicholls
+Nicholson Nixon Noble Nolan Norman Norris North Norton O'Blake O'Buckley
+O'Chamberlain O'Hobbs O'Thompson Oliver Osborne Owen Owens Page Palmer
+Parker Parkes Parkin Parkinson Parry Parsons Patel Patterson Payne
+Peacock Pearce Pearson Perkins Perry Peters Phillips Pickering Pollard
+Poole Pope Porter Potter Potts Powell Power Pratt Preston Price
+Pritchard Pugh Quinn Rahman Randall Read Reed Rees Reeves Reid Reynolds
+Rhodes Rice Richards Richardson Riley Roberts Robertson Robinson Robson
+Rogers Rose Ross Rowe Rowley Russell Ryan Sanders Sanderson Saunders
+Savage Schofield Scott Shah Sharp Sharpe Shaw Shepherd Sheppard Short
+Simmons Simpson Sims Sinclair Singh Skinner Slater Smart Smith Spencer
+Stanley Steele Stephens Stephenson Stevens Stevenson Stewart Stokes
+Stone Storey Sullivan Summers Sutton Swift Sykes Talbot Taylor Thomas
+Thomson Thornton Thorpe Todd Tomlinson Townsend Tucker Turnbull Turner
+Tyler Vaughan Vincent Wade Walker Wall Wallace Wallis Walsh Walters
+Walton Ward Warner Warren Waters Watkins Watson Watts Webb Webster Welch
+Wells West Weston Wheeler White Whitehead Whitehouse Whittaker Wilkins
+Wilkinson Williams Williamson Willis Wilson Winter Wong Wood Woods
+Woodward Wright Wyatt Yates Young
+""".strip().split()
+
+given_names_count = len(given_names)
+surnames_count = len(surnames)

django_db_anonymiser/database_sanitizer/session.py

@@ -0,0 +1,146 @@
+"""
+API to sanitation session.
+
+Sanitation session allows having a state within a single sanitation
+process.
+
+One important thing stored to the session is a secret key which is
+generated to a new random value for each sanitation session, but it
+stays constant during the whole sanitation process. Its value is never
+revealed, so that it is possible to generate such one way hashes with
+it, that should not be redoable afterwards. I.e. during the sanitation
+session it's possible to do ``hash(C) -> H`` for any clear text C, but
+it is not possible to check if H is the hashed value of C after the
+sanitation session has ended.
+"""
+
+import hashlib
+import hmac
+import random
+import sys
+import threading
+
+from six import int2byte
+
+if sys.version_info >= (3, 6):
+    from typing import Callable, Optional, Sequence  # noqa
+
+
+SECRET_KEY_BITS = 128
+
+
+_thread_local_storage = threading.local()
+
+
+def hash_text_to_int(value, bit_length=32):
+    # type: (str, int) -> int
+    """
+    Hash a text value to an integer.
+
+    Generates an integer number based on the hash derived with
+    `hash_text` from the given text value.
+
+    :param bit_length: Number of bits to use from the hash value.
+    :return: Integer value within ``0 <= result < 2**bit_length``
+    """
+    hash_value = hash_text(value)
+    return int(hash_value[0:(bit_length // 4)], 16)
+
+
+def hash_text_to_ints(value, bit_lengths=(16, 16, 16, 16)):
+    # type: (str, Sequence[int]) -> Sequence[int]
+    """
+    Hash a text value to a sequence of integers.
+
+    Generates a sequence of integer values with given bit-lengths
+    similarly to `hash_text_to_int`, but allowing generating many
+    separate numbers with a single call.
+
+    :param bit_lengths:
+      Tuple of bit lengths for the resulting integers.  Defines also the
+      length of the result tuple.
+    :return:
+      Tuple of ``n`` integers ``(R_1, ... R_n)`` with the requested
+      bit-lengths ``(L_1, ..., L_n)`` and values ranging within
+      ``0 <= R_i < 2**L_i`` for each ``i``.
+    """
+    hash_value = hash_text(value)
+    hex_lengths = [x // 4 for x in bit_lengths]
+    hex_ranges = (
+        (sum(hex_lengths[0:i]), sum(hex_lengths[0:(i + 1)]))
+        for i in range(len(hex_lengths)))
+    return tuple(int(hash_value[a:b], 16) for (a, b) in hex_ranges)
+
+
+def hash_text(value, hasher=hashlib.sha256, encoding='utf-8'):
+    # type: (str, Callable, str) -> str
+    """
+    Generate a hash for a text value.
+
+    The hash will be generated by encoding the text to bytes with given
+    encoding and then generating a hash with HMAC using the session
+    secret as the key and the given hash function.
+
+    :param value: Text value to hash
+    :param hasher: Hash function to use, SHA256 by default
+    :param encoding: Encoding to use, UTF-8 by default
+    :return: Hexadecimal presentation of the hash as a string
+    """
+    return hash_bytes(value.encode(encoding), hasher)
+
+
+def hash_bytes(value, hasher=hashlib.sha256):
+    # type: (bytes, Callable) -> str
+    """
+    Generate a hash for a bytes value.
+
+    The hash will be generated by generating a hash with HMAC using the
+    session secret as the key and the given hash function.
+
+    :param value: Bytes value to hash
+    :param hasher: Hash function to use.
+    :return: Hexadecimal presentation of the hash as a string
+    """
+    return hmac.new(get_secret(), value, hasher).hexdigest()
+
+
+def get_secret():
+    # type: () -> bytes
+    """
+    Get session specific secret key.
+
+    :return: Session key as bytes
+    """
+    if not getattr(_thread_local_storage, 'secret_key', None):
+        _initialize_session()
+    return _thread_local_storage.secret_key  # type: ignore
+
+
+def reset(secret_key=None):
+    # type: (Optional[bytes]) -> None
+    """
+    Reset the session.
+
+    By default, this resets the value of the secret to None so that, if
+    there was an earlier sanitation process ran on the same thread, then
+    a next call that needs the secret key of the session will generate a
+    new value for it.
+
+    This may also be used to set a predefined value for the secret key.
+
+    :param secret_key:
+      Value to set as the new session secret key or None if a new one
+      should be generated as soon as one is needed.
+    """
+    _thread_local_storage.secret_key = secret_key
+
+
+def _initialize_session():
+    # type: () -> None
+    """
+    Generate a new session key and store it to thread local storage.
+    """
+    sys_random = random.SystemRandom()
+    _thread_local_storage.secret_key = b''.join(
+        int2byte(sys_random.randint(0, 255))
+        for _ in range(SECRET_KEY_BITS // 8))