django-db-anonymiser 0.1.0__py3-none-any.whl

django_db_anonymiser/db_anonymiser/tests/test_faker.py

@@ -0,0 +1,116 @@
+import pytest
+
+from faker import Faker
+
+from db_anonymiser import faker as db_anonymiser_faker
+
+
+# Force faker to use a seed when producing output so that we can assume a
+# deterministic set of results
+@pytest.fixture(autouse=True)
+def seed_faker():
+    Faker.seed(0)
+
+
+def test_sanitize_name():
+    assert db_anonymiser_faker.sanitize_name("Bob Benson") == "Dr Rhys Thomas"
+
+
+def test_sanitize_first_name():
+    assert db_anonymiser_faker.sanitize_first_name("Bob") == "Emma"
+
+
+def test_sanitize_last_name():
+    assert db_anonymiser_faker.sanitize_last_name("Benson") == "Watts"
+
+
+def test_sanitize_email():
+    assert (
+        db_anonymiser_faker.sanitize_email("bob.benson@example.net")
+        == "areed@example.org"
+    )
+
+
+def test_sanitize_company_name():
+    assert db_anonymiser_faker.sanitize_company_name("Sterling Cooper") == "Reed-Burton"
+
+
+def test_sanitize_phone_number():
+    assert db_anonymiser_faker.sanitize_phone_number("011111111111") == "+448764759382"
+
+
+def test_sanitize_address():
+    assert (
+        db_anonymiser_faker.sanitize_address("Madison Ave")
+        == "Flat 4, Gibbons tunnel, Lesleystad, L8C 2EZ"
+    )
+
+
+def test_sanitize_website():
+    assert (
+        db_anonymiser_faker.sanitize_website("sterling.cooper")
+        == "reed-burton.arnold-jones.com"
+    )
+
+
+def test_sanitize_text():
+    assert (
+        db_anonymiser_faker.sanitize_text("Not great, Bob.")
+        == "Aliquam vitae laborum ullam rerum voluptas. Nesciunt tenetur magnam eligendi quidem nulla. Voluptates minus provident nobis corporis. Quas tempore placeat iusto. Explicabo et odit dignissimos."
+    )
+    assert db_anonymiser_faker.sanitize_text(None) == None
+
+
+def test_sanitize_street_address():
+    assert (
+        db_anonymiser_faker.sanitize_street_address("Madison Ave")
+        == "Studio 0\nArnold oval"
+    )
+
+
+def test_sanitize_city():
+    assert db_anonymiser_faker.sanitize_city("New York") == "Reedchester"
+
+
+def test_sanitize_postcode():
+    assert db_anonymiser_faker.sanitize_postcode("DT11 7DY") == "TA60 8UR"
+
+
+def test_sanitize_eori_number():
+    assert (
+        db_anonymiser_faker.sanitize_eori_number("GB111111111111") == "GB424533559245"
+    )
+
+
+def test_sanitize_ni_eori_number():
+    assert (
+        db_anonymiser_faker.sanitize_ni_eori_number("XI111111111111") == "XI424533559245"
+    )
+
+def test_sanitize_eu_eori_number():
+    assert (
+        db_anonymiser_faker.sanitize_eu_eori_number("FR11111") == "FR99346"
+    )
+
+def test_sanitize_sic_number():
+    assert db_anonymiser_faker.sanitize_sic_number("11111") == "50494"
+
+
+def test_sanitize_vat_number():
+    assert db_anonymiser_faker.sanitize_vat_number("GB111111111") == "GB906691059"
+
+
+def test_sanitize_registration_number():
+    assert db_anonymiser_faker.sanitize_registration_number("11111111") == "51706749"
+
+
+def test_sanitize_filename():
+    assert db_anonymiser_faker.sanitize_filename("somefile.txt") == "molestiae.xlsx"
+
+
+def test_sanitize_short_text():
+    assert (
+        db_anonymiser_faker.sanitize_short_text("some small text")
+        == "Tempore placeat iusto aut. Et odit dignissimos mollitia ipsam maxime."
+    )
+    assert db_anonymiser_faker.sanitize_short_text(None) == None

django_db_anonymiser-0.1.0.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.3
+Name: django-db-anonymiser
+Version: 0.1.0
+Summary: Django app to create configurable anonymised DB dumps.
+Author: Brendan Smith
+Author-email: brendan.smith@digital.trade.gov.uk
+Requires-Python: >3.9.1,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Requires-Dist: boto3 (>=1.40.33,<2.0.0)
+Requires-Dist: django (>=4.2.10,<5.0.0)
+Requires-Dist: django-environ (>=0.12.0,<0.13.0)
+Requires-Dist: faker (>=4.18.0)
+Requires-Dist: psycopg2-binary (>=2.9.10,<3.0.0)
+Requires-Dist: pymysql (>=1.1.2,<2.0.0)
+Description-Content-Type: text/markdown
+
+# django-db-anonymiser
+Django app to create configurable anonymised DB dumps.
+
+django-db-anonymiser provides a django app with a management command `dump_and_anonymise`.
+This command runs a `pg_dump` against a postgresql DB, applies anonymisation functions to 
+data dumped from the DB and then writes the anonymised dump to S3.
+See here for lite-api's example anonymisation configuration; https://github.com/uktrade/lite-api/blob/dev/api/conf/anonymise_model_config.yaml
+
+This pattern is designed as a replacement for Lite's old DB anonymisation process (although it is general purpose and can be used for any django project which uses postgresql).
+The previous process was baked in to an airflow installation and involved making
+a `pg_dump` from production, anonymising that dump with python and pushing the 
+file to S3. See; https://github.com/uktrade/lite-airflow-dags/blob/master/dags/export_lite_db.py
+
+django-db-anonymiser follows the same overall pattern, but aims to achieve it
+through a django management command instead of running on top of airflow.  In addition,
+the configuration for how DB columns are anonymised can be configured in simple YAML.
+
+**Note:** This repository depends upon code forked from https://github.com/andersinno/python-database-sanitizer
+This is housed under the `database_sanitizer` directory and has been forked from the above repository 
+because it is unmaintained.
+
+## Getting started
+
+- Add `faker>=4.18.0`, `boto3>=1.26.17` to python requirements; it is assumed python/psycopg and co are already installed.
+- Either add this github repository as a submodule to your django application named `django_db_anonymiser` or install the python package (django-db-anonymiser)[] from PyPI.
+- Add `django_db_anonymiser.db_anonymiser` to `INSTALLED_APPS`
+- Set the following django settings;
+    - `DB_ANONYMISER_CONFIG_LOCATION` - the location of your anonymisation yaml file
+    - `DB_ANONYMISER_AWS_ENDPOINT_URL` - optional, custom URL for AWS (e.g. if using minio)
+    - `DB_ANONYMISER_AWS_ACCESS_KEY_ID` - AWS access key ID for the S3 bucket to upload dumps to
+    - `DB_ANONYMISER_AWS_SECRET_ACCESS_KEY` - AWS secret key for the S3 bucket to upload dumps to
+    - `DB_ANONYMISER_AWS_REGION` - AWS region for the S3 bucket to upload dumps to
+    - `DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME` - AWS bucket name for the S3 bucket to upload dumps to
+
+## Running tests
+
+For local unit testing from the root of the repository run:
+
+    $ poetry run pytest
+
+**Note:** Currently for full test coverage, it is necessary to run tests in circleci, where we spin up a postgres db and test
+the `db_anonymiser` command directly
+
+## Publishing
+
+Publishing to PyPI is currently a manual process:
+
+1. Acquire API token from [Passman](https://passman.ci.uktrade.digital/secret/0f3d699a-1c7a-4e92-a235-6c756f678dd5/).
+   - Request access from the SRE team.
+   - _Note: You will need access to the `platform` group in Passman._
+2. Run `poetry config pypi-token.pypi <token>` to add the token to your Poetry configuration.
+
+Update the version, as the same version cannot be published to PyPI.
+
+```
+poetry version patch
+```
+
+More options for the `version` command can be found in the [Poetry documentation](https://python-poetry.org/docs/cli/#version). For example, for a minor version bump: `poetry version minor`.
+
+Build the Python package.
+
+```
+poetry build
+```
+
+Publish the Python package.
+
+_Note: Make sure your Pull Request (PR) is approved and contains the version upgrade in `pyproject.toml` before publishing the package._
+
+```
+poetry publish
+```
+
+Check the [PyPI Release history](https://pypi.org/project/dbt-platform-helper/#history) to make sure the package has been updated.
+
+For an optional manual check, install the package locally and test everything works as expected.
+

django_db_anonymiser-0.1.0.dist-info/RECORD

@@ -0,0 +1,40 @@
+django_db_anonymiser/database_sanitizer/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/database_sanitizer/__main__.py,sha256=MD5YHVO17s3v1A836StbLnjnbHe8RUwccC4emP5DmiQ,1551
+django_db_anonymiser/database_sanitizer/config.py,sha256=RBqzK8AU4Z9b5_BUIPQMsDCAEpZ9VKUxUHExu0GdtQc,13501
+django_db_anonymiser/database_sanitizer/dump/__init__.py,sha256=IEMKBXC0t6seCIWGw9hXVTS389u8_lUnFiGDl5jVIho,1575
+django_db_anonymiser/database_sanitizer/dump/mysql.py,sha256=b5DJKoY-5j_17beSmFFI3r31LgYyuqcwlHNY23E_YHI,6431
+django_db_anonymiser/database_sanitizer/dump/postgres.py,sha256=AiJWydA-ilX793waQBtpaBlgn2EuE3_nU-2V9fg1vh0,4988
+django_db_anonymiser/database_sanitizer/sanitizers/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/database_sanitizer/sanitizers/constant.py,sha256=0PHvKAF2NEPi2qZ-agnD_dOD3rKYccoMVV1i5BQXOO8,214
+django_db_anonymiser/database_sanitizer/sanitizers/derived.py,sha256=6UN3703pR5g7mcrwF-lxrfRmmtdhPTZfpkdqCpl_PuE,375
+django_db_anonymiser/database_sanitizer/sanitizers/string.py,sha256=h0VMsFgKwYBxqGgLMbdu2iP7rarFy1yqXKX_O8KLTmo,708
+django_db_anonymiser/database_sanitizer/sanitizers/times.py,sha256=ORa5MlwcgHNhwP2VIZNKkyTV3dU1_DYfmkG_COToqN0,299
+django_db_anonymiser/database_sanitizer/sanitizers/user.py,sha256=vj-CzSb5ckToTwGQ0eRf9u9cxk0dDXRafXfOVG1HHzI,7547
+django_db_anonymiser/database_sanitizer/session.py,sha256=gO5RV0zd4Qk4mayVKu4W1nqoAkX-pRDGNS-DWM6OcHU,4597
+django_db_anonymiser/database_sanitizer/tests/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/database_sanitizer/tests/test_config.py,sha256=1PC-sv9ZdN-kVnO6oFxxOg3LEJbTWSsaEQ0hGinatY4,8367
+django_db_anonymiser/database_sanitizer/tests/test_dump.py,sha256=0Gv1NvzApKE-5YgzIagvvHUkBC7ClnURuiAn_qjLj1o,3692
+django_db_anonymiser/database_sanitizer/tests/test_dump_mysql.py,sha256=Vg970aNuOuOizHvxPz5501EPaFvYnGAjFQFl0gOnmKw,5635
+django_db_anonymiser/database_sanitizer/tests/test_dump_postgres.py,sha256=qaX8RbaigwRINonATD7gYFMajuKgA32iTZ0x2cHsjnk,5483
+django_db_anonymiser/database_sanitizer/tests/test_main.py,sha256=hZr1iuiFhWhypctmp_RKFIW87aFDxleMjjFyrHkPta4,3161
+django_db_anonymiser/database_sanitizer/tests/test_sanitizers_constant.py,sha256=c0kdaTnEAY-uqlTOGmEpK8cINyuTTuLuVSIR1Yi74wI,1156
+django_db_anonymiser/database_sanitizer/tests/test_sanitizers_derived.py,sha256=Mh29lsKmM6skxYcM9hSkOF12EQg2mKpML8yZBdZJZdM,741
+django_db_anonymiser/database_sanitizer/tests/test_sanitizers_string.py,sha256=yafmFXob0n__LFlk52nuj3zqOU3ZWHbGd8N65c_SIUA,1077
+django_db_anonymiser/database_sanitizer/tests/test_sanitizers_times.py,sha256=m08KiGkms912PirRWJgkUQfPb_oxpbdIifOHAltIx4g,471
+django_db_anonymiser/database_sanitizer/tests/test_sanitizers_user.py,sha256=g-qBahyxs2_MYy9QVelt1mdKYYIsCP6bYwIO1mAqgm8,2792
+django_db_anonymiser/database_sanitizer/tests/test_session.py,sha256=l4jTJu5wJV6HemoabKZFO0_ZFICngTC7L_Z0JUCoGV8,906
+django_db_anonymiser/database_sanitizer/tests/test_utils_mysql.py,sha256=jSrYZdInPdKNQD9f3yphDaGHWMmnjSskLYV2wZ1ooWI,2737
+django_db_anonymiser/database_sanitizer/tests/test_utils_postgres.py,sha256=6duk1llpfmU5pf0S3lxDLCiMHgucUB4mpK567Udm27Q,2266
+django_db_anonymiser/database_sanitizer/utils/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/database_sanitizer/utils/mysql.py,sha256=Pg6Qv2W6v7FgEaxcLGUZVDoJDwAYv7hxCQPcfOFZzL8,4657
+django_db_anonymiser/database_sanitizer/utils/postgres.py,sha256=v2rm3dX_9Z7NdkY8irJ-rMEh6i31o8ymEqpxHSdtaqU,4245
+django_db_anonymiser/db_anonymiser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/db_anonymiser/faker.py,sha256=5X8tH_MAMtrsg248waXMSXTkoacMosqQlkCSFjy_0pA,1729
+django_db_anonymiser/db_anonymiser/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/db_anonymiser/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+django_db_anonymiser/db_anonymiser/management/commands/dump_and_anonymise.py,sha256=XNyDLwGgDWD1vXUs9ChYPdzI-gbLOJrcfcCceRR1tCU,3692
+django_db_anonymiser/db_anonymiser/tests/test_command.py,sha256=lngzFc-_K2fC-LgIaLNSkJa3iBrDAPWxh56heIa5whg,3637
+django_db_anonymiser/db_anonymiser/tests/test_faker.py,sha256=gvrOpSlWDjr4ajLyN5BEhxtkxO7_zls-CP04EolFFV4,3197
+django_db_anonymiser-0.1.0.dist-info/METADATA,sha256=G1Wk_yRHC2lIfUXUsrk1pBf2frmBM0dCz29eGujFFCc,4567
+django_db_anonymiser-0.1.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+django_db_anonymiser-0.1.0.dist-info/RECORD,,

django_db_anonymiser-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 2.1.3
+Root-Is-Purelib: true
+Tag: py3-none-any