django-db-anonymiser 0.2.5__py3-none-any.whl → 0.3.0__py3-none-any.whl

django_db_anonymiser/db_anonymiser/management/commands/dump_and_anonymise.py

@@ -1,5 +1,6 @@
 import os
 import logging
+from datetime import datetime
 
 import boto3
 from django.conf import settings
@@ -28,16 +29,30 @@ class Command(BaseCommand):
             action="store_true",
             help="Generates and logs a presigned URL for the uploaded file.",
         )
+        parser.add_argument(
+            "--add-timestamp",
+            action="store_true",
+            help="Add current timestamp to dump file name.",
+        )
 
     def configure(self):
         self.keep_local_dumpfile = False
         self.skip_s3_upload = False
         self.presign = False
-        self.dump_file_name = settings.DB_ANONYMISER_DUMP_FILE_NAME
+        base_dump_file_name = settings.DB_ANONYMISER_DUMP_FILE_NAME
+        if self.add_timestamp:
+            now = datetime.now().strftime("%Y-%m-%d-%H:%M:%S")
+            base_dump_file_name = f"{now}-{base_dump_file_name}"
+        dump_file_name = base_dump_file_name
+        if getattr(settings, "DB_ANONYMISER_AWS_STORAGE_KEY", None):
+            dump_file_name = (
+                f"{settings.DB_ANONYMISER_AWS_STORAGE_KEY}/{dump_file_name}"
+            )
+        self.dump_file_name = dump_file_name
         self.temporary_dump_location = getattr(
             settings,
             "DB_ANONYMISER_TEMPORARY_DUMP_LOCATION",
-            f"/tmp/{self.dump_file_name}",
+            f"/tmp/{base_dump_file_name}",
         )
         try:
             self.config_location = settings.DB_ANONYMISER_CONFIG_LOCATION
@@ -62,6 +77,11 @@ class Command(BaseCommand):
 
     def handle(self, *args, **options):
         logger.info("Starting DB dump and anonymiser")
+
+        self.add_timestamp = False
+        if options["add_timestamp"]:
+            self.add_timestamp = True
+
         self.configure()
 
         if options["keep_local_dumpfile"]:
@@ -69,7 +89,7 @@ class Command(BaseCommand):
 
         if options["skip_s3_upload"]:
             self.skip_s3_upload = True
-        
+
         if options["presign"]:
             self.presign = True
 
@@ -104,12 +124,12 @@ class Command(BaseCommand):
             self.temporary_dump_location, self.s3_bucket_name, self.dump_file_name
         )
         logger.info("Writing file to S3 complete")
-    
+
     def generate_presigned_url(self):
         presigned = self.s3_client.generate_presigned_url(
-            'get_object',
-            Params={'Bucket': self.s3_bucket_name, 'Key': self.dump_file_name},
-            ExpiresIn=600
+            "get_object",
+            Params={"Bucket": self.s3_bucket_name, "Key": self.dump_file_name},
+            ExpiresIn=600,
         )
         logger.info("Presigned URL: %s", presigned)
 

django_db_anonymiser/db_anonymiser/tests/test_command.py

@@ -1,11 +1,12 @@
 import os
+from datetime import datetime
 from unittest.mock import ANY
 from unittest.mock import patch
 
 from django.conf import settings
 from django.contrib.auth.models import User
 from django.core.management import call_command
-from django.test import TransactionTestCase
+from django.test import override_settings, TransactionTestCase
 
 import boto3
 import pytest
@@ -13,7 +14,10 @@ import requests
 from moto import mock_aws
 
 
-@pytest.mark.skipif(os.getenv("CIRCLECI") != "true", reason="Skipped because test requires real postgres db.")
+@pytest.mark.skipif(
+    os.getenv("CIRCLECI") != "true",
+    reason="Skipped because test requires real postgres db.",
+)
 @mock_aws
 class TestDumpAndAnonmyiseCommand(TransactionTestCase):
     def setUp(self):
@@ -91,12 +95,18 @@ class TestDumpAndAnonmyiseCommand(TransactionTestCase):
         call_command("dump_and_anonymise", keep_local_dumpfile=True)
         assert not mocked_os_remove.called
 
-    @patch("django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.logger")
-    @patch("django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.Command.generate_presigned_url")
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.logger"
+    )
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.Command.generate_presigned_url"
+    )
     @patch(
         "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.os.remove"
     )
-    def test_dump_and_anonymise_no_arguments(self, mocked_os_remove, mocked_presign, mocked_logger):
+    def test_dump_and_anonymise_no_arguments(
+        self, mocked_os_remove, mocked_presign, mocked_logger
+    ):
         call_command("dump_and_anonymise")
         bucket_contents = self.aws.list_objects(
             Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
@@ -107,10 +117,15 @@ class TestDumpAndAnonmyiseCommand(TransactionTestCase):
         )
         mocked_presign.assert_not_called()
         mocked_logger.info.assert_any_call("DB dump and anonymiser was successful!")
-        mocked_logger.info.assert_any_call("Writing anonymised dumpfile to temporary location %s", settings.DB_ANONYMISER_DUMP_FILE_NAME)
+        mocked_logger.info.assert_any_call(
+            "Writing anonymised dumpfile to temporary location %s",
+            settings.DB_ANONYMISER_DUMP_FILE_NAME,
+        )
         mocked_logger.info.assert_any_call("Writing file to S3 complete")
 
-    @patch("django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.logger")
+    @patch(
+        "django_db_anonymiser.db_anonymiser.management.commands.dump_and_anonymise.logger"
+    )
     def test_dump_and_anonymise_with_presign(self, mocked_logger):
         call_command("dump_and_anonymise", presign=True)
         bucket_contents = self.aws.list_objects(
@@ -120,3 +135,28 @@ class TestDumpAndAnonmyiseCommand(TransactionTestCase):
         mocked_logger.info.assert_any_call("Presigned URL: %s", ANY)
         presigned_url = mocked_logger.info.call_args_list[6][0][1]
         assert requests.get(presigned_url).status_code == 200
+
+    def test_dump_and_anonymise_with_timestamp(self):
+        now = datetime.now()
+        patch(
+            "django_db_anonymiser.db_anonymiser.management.commands.datetime.datetime.now",
+            return_value=now,
+        )
+        call_command("dump_and_anonymise", add_timestamp=True)
+        bucket_contents = self.aws.list_objects(
+            Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
+        ).get("Contents", [])
+        assert (
+            bucket_contents[0]["Key"]
+            == f"{now.strftime('%Y-%m-%d-%H:%M:%S')}-{settings.DB_ANONYMISER_DUMP_FILE_NAME}"
+        )
+
+    @override_settings(DB_ANONYMISER_AWS_STORAGE_KEY="test")
+    def test_dump_and_anonymise_with_s3_key(self):
+        call_command("dump_and_anonymise")
+        bucket_contents = self.aws.list_objects(
+            Bucket=settings.DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME
+        ).get("Contents", [])
+        assert (
+            bucket_contents[0]["Key"] == f"test/{settings.DB_ANONYMISER_DUMP_FILE_NAME}"
+        )

{django_db_anonymiser-0.2.5.dist-info → django_db_anonymiser-0.3.0.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.3
 Name: django-db-anonymiser
-Version: 0.2.5
+Version: 0.3.0
 Summary: Django app to create configurable anonymised DB dumps.
 Author: Brendan Smith
 Author-email: brendan.smith@digital.trade.gov.uk
@@ -22,13 +22,13 @@ Description-Content-Type: text/markdown
 Django app to create configurable anonymised DB dumps.
 
 django-db-anonymiser provides a django app with a management command `dump_and_anonymise`.
-This command runs a `pg_dump` against a postgresql DB, applies anonymisation functions to 
+This command runs a `pg_dump` against a postgresql DB, applies anonymisation functions to
 data dumped from the DB and then writes the anonymised dump to S3.
 See here for lite-api's example anonymisation configuration; https://github.com/uktrade/lite-api/blob/dev/api/conf/anonymise_model_config.yaml
 
 This pattern is designed as a replacement for Lite's old DB anonymisation process (although it is general purpose and can be used for any django project which uses postgresql).
 The previous process was baked in to an airflow installation and involved making
-a `pg_dump` from production, anonymising that dump with python and pushing the 
+a `pg_dump` from production, anonymising that dump with python and pushing the
 file to S3. See; https://github.com/uktrade/lite-airflow-dags/blob/master/dags/export_lite_db.py
 
 django-db-anonymiser follows the same overall pattern, but aims to achieve it
@@ -36,7 +36,7 @@ through a django management command instead of running on top of airflow.  In ad
 the configuration for how DB columns are anonymised can be configured in simple YAML.
 
 **Note:** This repository depends upon code forked from https://github.com/andersinno/python-database-sanitizer
-This is housed under the `database_sanitizer` directory and has been forked from the above repository 
+This is housed under the `database_sanitizer` directory and has been forked from the above repository
 because it is unmaintained.
 
 ## Getting started
@@ -51,6 +51,8 @@ because it is unmaintained.
     - `DB_ANONYMISER_AWS_SECRET_ACCESS_KEY` - AWS secret key for the S3 bucket to upload dumps to
     - `DB_ANONYMISER_AWS_REGION` - AWS region for the S3 bucket to upload dumps to
     - `DB_ANONYMISER_AWS_STORAGE_BUCKET_NAME` - AWS bucket name for the S3 bucket to upload dumps to
+    - `DB_ANONYMISER_DUMP_FILE_NAME` - Name for dumped DB file
+    - `DB_ANONYMISER_AWS_STORAGE_KEY` - optional, key under which file will be stored in AWS S3 bucket
 
 ## Running tests
 
@@ -65,7 +67,7 @@ the `db_anonymiser` command directly
 
 Publishing to PyPI is currently a manual process:
 
-1. Acquire API token from [Passman](https://passman.ci.uktrade.digital/secret/0f3d699a-1c7a-4e92-a235-6c756f678dd5/).
+1. Acquire API token from [Passman](https://passman.ci.uktrade.digital/secret/0f3d699a-1c7a-4e92-a235-6c756f678dd5/). <!-- /PS-IGNORE -->
    - Request access from the SRE team.
    - _Note: You will need access to the `platform` group in Passman._
 2. Run `poetry config pypi-token.pypi <token>` to add the token to your Poetry configuration.

{django_db_anonymiser-0.2.5.dist-info → django_db_anonymiser-0.3.0.dist-info}/RECORD

@@ -32,9 +32,9 @@ django_db_anonymiser/db_anonymiser/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQe
 django_db_anonymiser/db_anonymiser/faker.py,sha256=5X8tH_MAMtrsg248waXMSXTkoacMosqQlkCSFjy_0pA,1729
 django_db_anonymiser/db_anonymiser/management/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 django_db_anonymiser/db_anonymiser/management/commands/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-django_db_anonymiser/db_anonymiser/management/commands/dump_and_anonymise.py,sha256=kvocsusmjuBeBKfKE9xg9k3MdnY6hUBT__jc6Q2HHYc,4339
-django_db_anonymiser/db_anonymiser/tests/test_command.py,sha256=J6_KF59S-DBtOx5yumP1Y1LRYU2jR78R0-oOupEflB0,5481
+django_db_anonymiser/db_anonymiser/management/commands/dump_and_anonymise.py,sha256=Zoe-VuBRKaTV37-60zim8z4FofAM-OiQ0wToh-zQE9w,5074
+django_db_anonymiser/db_anonymiser/tests/test_command.py,sha256=mO35e7MqfebXYa0NzXFVmBvAPgzOpEwkWjdwWuxvmhE,6647
 django_db_anonymiser/db_anonymiser/tests/test_faker.py,sha256=gvrOpSlWDjr4ajLyN5BEhxtkxO7_zls-CP04EolFFV4,3197
-django_db_anonymiser-0.2.5.dist-info/METADATA,sha256=88G-8EQVIh-t5lLg3L0mES2J9Sd-is_NIJ7fr6HQyms,4617
-django_db_anonymiser-0.2.5.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
-django_db_anonymiser-0.2.5.dist-info/RECORD,,
+django_db_anonymiser-0.3.0.dist-info/METADATA,sha256=gkxgkQ8oqUS_0ugDmbUBmVaHsADMdo-Max_in544Qao,4800
+django_db_anonymiser-0.3.0.dist-info/WHEEL,sha256=b4K_helf-jlQoXBBETfwnf4B04YC67LOev0jo4fX5m8,88
+django_db_anonymiser-0.3.0.dist-info/RECORD,,