idscrub 1.0.1__tar.gz

idscrub-1.0.1/.github/pull_request_template.md

@@ -0,0 +1,27 @@
+## What
+
+<!---
+What is this PR doing, e.g. implementations, algorithms, etc.?
+ * Set the scene - you probably have a lot of context in your head that the reader doesn't have.
+ * Explain like I'm 5 - try to make as few assumptions as possible about the reader
+ * Use pictures, screenshots, or a diagram if you can, for example https://docs.github.com/en/get-started/writing-on-github/working-with-advanced-formatting/creating-diagrams#creating-mermaid-diagrams
+--->
+
+## Why
+
+<!---
+Why is this change happening, e.g. goals, use cases, stories, etc.?
+ * Explain what the problem was that this PR addresses.
+ * Explain why this solution was chosen, and any alternatives considered.
+ * Mention any assumptions, deliberately ignored edge-cases, or changes that are left for later.
+--->
+
+## How this has been tested
+
+- [ ] I have tested locally
+- [ ] I have added a new unit test (if appropriate)
+- [ ] Testing not required
+
+## Reviewer Checklist
+
+- [ ] I have reviewed the PR and ensured no secret values are present

idscrub-1.0.1/.github/workflows/cd.yml

@@ -0,0 +1,35 @@
+name: "Publish"
+
+on:
+  push:
+    tags:
+      # Publish on any tag starting with a `v`, e.g., v0.1.0
+      - v*
+
+jobs:
+  run:
+    runs-on: ubuntu-latest
+    environment:
+      name: pypi
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v5
+      - name: Install uv
+        uses: astral-sh/setup-uv@v7
+
+      - name: Install Python 3.12
+        run: uv python install 3.12
+      - name: Build
+        run: uv build
+
+      # Check that basic features work and we didn't miss to include crucial files
+      - name: Regex test (wheel)
+        run: uv run --isolated --no-project --with dist/*.whl test/test_regex.py
+      - name: Regex test (source distribution)
+        run: uv run --isolated --no-project --with dist/*.tar.gz test/test_regex.py
+        
+      - name: Publish
+        run: uv publish

idscrub-1.0.1/.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: test 
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv and Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --frozen --all-extras --dev
+
+      - name: Run Python tests
+        run: uv run pytest
+
+      - name: Dump docker logs
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+  scan-for-secrets:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Run TruffleHog scan
+        run: |
+          docker run --rm -v ${{ github.workspace }}:/repo \
+            trufflesecurity/trufflehog:latest \
+            git file:///repo \
+            --since-commit main \
+            --branch ${{ github.head_ref || github.ref_name }} \
+            --github-actions \
+            --fail
+
+  build-package:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Install uv and Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build package
+        run: uv build
+
+  ci-success:
+    needs: [test, scan-for-secrets, build-package]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    
+    steps:
+      - name: CI success
+        run: echo "All CI checks passed!"

idscrub-1.0.1/.gitignore

@@ -0,0 +1,18 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# idscrub specific
+huggingface
+
+.pytest_cache
+.ruff_cache
+.coverage
+.DS_Store

idscrub-1.0.1/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+- repo: local
+  hooks:
+    # Run the formatter.
+    - id: ruff
+      name: ruff
+      description: "Lint and format code using Ruff"
+      entry: make format
+      language: system
+      types_or: [python, pyi]
+      pass_filenames: false
+
+# Mandatory internal hooks
+- repo: https://github.com/uktrade/github-standards
+  rev: v1.2.1  # update periodically with pre-commit autoupdate
+  hooks:
+    - id: run-security-scan
+      verbose: false
+    - id: run-personal-data-scan
+      verbose: false
+    - id: validate-security-scan
+      verbose: false

idscrub-1.0.1/CODEOWNERS

@@ -0,0 +1 @@
+* @uktrade/ag-data-science

idscrub-1.0.1/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Department for Business and Trade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

idscrub-1.0.1/Makefile

@@ -0,0 +1,15 @@
+.PHONY: clean precommit test 
+
+## Delete all compiled Python files
+clean:
+	find . -type f -name "*.py[co]" -delete
+	find . -type d -name "__pycache__" -delete
+
+## Reformat, lint
+format:
+	uv run ruff format .
+	uv run ruff check . --fix
+
+## Run Python tests
+test:
+	uv run pytest

idscrub-1.0.1/PKG-INFO

@@ -0,0 +1,149 @@
+Metadata-Version: 2.4
+Name: idscrub
+Version: 1.0.1
+Author: Department for Business and Trade
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ipykernel>=7.1.0
+Requires-Dist: ipywidgets
+Requires-Dist: numpy>=2.3.4
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: phonenumbers>=9.0.18
+Requires-Dist: pip>=25.3
+Requires-Dist: spacy-transformers>=1.3.9
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: presidio-analyzer
+Requires-Dist: presidio-anonymizer
+Provides-Extra: trf
+Requires-Dist: en_core_web_trf; extra == "trf"
+Dynamic: license-file
+
+# idscrub 🧽✨
+
+* Names and other personally identifying information are often present in text, even if they are not clearly visible or requested.
+* This information may need to be removed prior to further analysis in many cases.
+* `idscrub` identifies and removes (*✨scrubs✨*) personal data from text using [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) and [named-entity recognition](https://en.wikipedia.org/wiki/Named-entity_recognition).
+
+## Installation
+
+`idscrub` can be installed using `pip` into a Python **>=3.12** environment. Example:
+
+```console
+pip install idscrub
+```
+or with the spaCy transformer model (`en_core_web_trf`) already installed:
+
+```console
+pip install idscrub[trf]
+```
+## How to use the code
+
+Basic usage example (see [basic_usage.ipynb](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) for further examples):
+
+```python
+from idscrub import IDScrub
+
+scrub = IDScrub(['Our names are Hamish McDonald, L. Salah, and Elena Suárez.', 'My number is +441111111111 and I live at AA11 1AA.'])x
+scrubbed_texts = scrub.scrub(scrub_methods=['spacy_persons', 'uk_phone_numbers', 'uk_postcodes'])
+
+print(scrubbed_texts)
+
+# Output: ['Our names are [PERSON], [PERSON], and [PERSON].', 'My number is [PHONENO] and I live at [POSTCODE].']
+```
+## Personal data types supported
+
+Personal data can either be scrubbed as methods with arguments for extra customisation, e.g. `IDScrub.google_phone_numbers(region="GB")`, or as a string arguments with default configurations (see above). The method name and its string representation are the same. 
+
+| Argument                | Scrubs                                                                 |
+|-------------------------|------------------------------------------------------------------------|
+| `all`                  | All supported personal data types (see `IDScrub.all()` for further customisation)                                      |
+| `spacy_persons`        | Person names detected by spaCy's `en_core_web_trf` (or other user-selected spaCy models)                                    |
+| `huggingface_persons`  | Person names detected by user-selected HuggingFace models                        |
+| `email_addresses`      | Email addresses                                                       |
+| `titles`               | Titles (e.g., Mr., Mrs., Dr.)                                         |
+| `handles`              | Social media handles (e.g., @username)                                |
+| `ip_addresses`         | IP addresses                                                          |
+| `uk_postcodes`         | UK postal codes                                                       |
+| `uk_phone_numbers`     | UK phone numbers                                                      |
+| `google_phone_numbers` | Phone numbers detected by Google’s [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) |
+| `presidio`             | Entities supported by [Microsoft Presidio](https://microsoft.github.io/presidio/) (e.g., names, URLs, NHS numbers, IBAN codes) |
+
+## Considerations before use
+
+- You must follow [GDPR guidance](https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/the-research-provisions/principles-and-grounds-for-processing/) when processing personal data using this package.
+- This package has been designed as a *first pass* for standardised personal data removal. 
+- Users are encouraged to check and confirm outputs and conduct manual reviews where necessary, e.g. when cleaning high risk datasets.
+- It is up to the user to assess whether this removal process needs to be supplemented by other methods for their given dataset and security requirements.
+
+### Input data
+
+- This package is designed for text-based documents structured as a list of strings.
+- It performs best when contextual meaning can be inferred from the text.
+- For best results, input text should therefore resemble natural language. 
+- **Highly fragmented, informal, technical, or syntactically broken text may reduce detection accuracy and lead to incomplete or incorrect name detection.**
+
+### Biases and evaluation
+
+- `idscrub` supports integration with SpaCy and Hugging Face models for name cleaning.
+- These models are state-of-the-art, capable of identifying approximately 90% of named entities, but **may not remove all names**.
+- **Biases present in these models due to their training data may affect performance**. For example:
+    - English names may be more reliably identified than names common in other languages.
+    - Uncommon or non-Western naming conventions may be missed or misclassified.
+
+> [!IMPORTANT]
+> * See [our wiki](https://github.com/uktrade/idscrub/wiki/Evaluation) for further details and notes on our evaluation of `idscrub`.
+
+### Models
+
+* Only Spacy's `en_core_web_trf` and no Hugging Face models have been formally evaluated.
+* We therefore recommend that the current default `en_core_web_trf` is used for name scrubbing. **Other models need to be evaluated by the user.**
+
+## Similar Python packages
+
+* Similar packages exist for undertaking this task, such as [Presidio](https://microsoft.github.io/presidio/), [Scrubadub](https://github.com/LeapBeyond/scrubadub) and [Sanityze](https://github.com/UBC-MDS/sanityze). 
+* Development of `idscrub` was undertaken to: 
+
+    * Bring together different scrubbing methods across the Department for Business and Trade.
+    * Adhere to infrastructure requirements.
+    * Guarantee future stability and maintainability.
+    * Encourage future scrubbing methods to be added collaboratively and transparently.
+    * Allow for full flexibility depending on the use case and required outputs.
+    
+* To leverage the power of other packages, we have added methods that allow you to interact with them. These include: `IDScrub.presidio()` and `IDScrub.google_phone_numbers()`. See the [usage example notebook](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) and method docstrings for further information.
+
+## AI declaration
+
+AI has been used in the development of `idscrub`, primarily to develop regular expressions, suggest code refinements and draft documentation.
+
+## Development setup
+
+This project is managed by [uv](https://docs.astral.sh/uv/).
+
+To install all dependencies for this project, run:
+
+```console
+uv sync --all-extras
+```
+
+If you do not have Python 3.12, run:
+
+```console
+uv python install 3.12
+```
+
+To run tests:
+
+```console
+uv run pytest
+```
+
+or
+
+```console
+make test
+```
+
+## Author 
+
+Analytical Data Science, Department for Business and Trade

idscrub-1.0.1/README.md

@@ -0,0 +1,128 @@
+# idscrub 🧽✨
+
+* Names and other personally identifying information are often present in text, even if they are not clearly visible or requested.
+* This information may need to be removed prior to further analysis in many cases.
+* `idscrub` identifies and removes (*✨scrubs✨*) personal data from text using [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) and [named-entity recognition](https://en.wikipedia.org/wiki/Named-entity_recognition).
+
+## Installation
+
+`idscrub` can be installed using `pip` into a Python **>=3.12** environment. Example:
+
+```console
+pip install idscrub
+```
+or with the spaCy transformer model (`en_core_web_trf`) already installed:
+
+```console
+pip install idscrub[trf]
+```
+## How to use the code
+
+Basic usage example (see [basic_usage.ipynb](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) for further examples):
+
+```python
+from idscrub import IDScrub
+
+scrub = IDScrub(['Our names are Hamish McDonald, L. Salah, and Elena Suárez.', 'My number is +441111111111 and I live at AA11 1AA.'])x
+scrubbed_texts = scrub.scrub(scrub_methods=['spacy_persons', 'uk_phone_numbers', 'uk_postcodes'])
+
+print(scrubbed_texts)
+
+# Output: ['Our names are [PERSON], [PERSON], and [PERSON].', 'My number is [PHONENO] and I live at [POSTCODE].']
+```
+## Personal data types supported
+
+Personal data can either be scrubbed as methods with arguments for extra customisation, e.g. `IDScrub.google_phone_numbers(region="GB")`, or as a string arguments with default configurations (see above). The method name and its string representation are the same. 
+
+| Argument                | Scrubs                                                                 |
+|-------------------------|------------------------------------------------------------------------|
+| `all`                  | All supported personal data types (see `IDScrub.all()` for further customisation)                                      |
+| `spacy_persons`        | Person names detected by spaCy's `en_core_web_trf` (or other user-selected spaCy models)                                    |
+| `huggingface_persons`  | Person names detected by user-selected HuggingFace models                        |
+| `email_addresses`      | Email addresses                                                       |
+| `titles`               | Titles (e.g., Mr., Mrs., Dr.)                                         |
+| `handles`              | Social media handles (e.g., @username)                                |
+| `ip_addresses`         | IP addresses                                                          |
+| `uk_postcodes`         | UK postal codes                                                       |
+| `uk_phone_numbers`     | UK phone numbers                                                      |
+| `google_phone_numbers` | Phone numbers detected by Google’s [phonenumbers](https://github.com/daviddrysdale/python-phonenumbers) |
+| `presidio`             | Entities supported by [Microsoft Presidio](https://microsoft.github.io/presidio/) (e.g., names, URLs, NHS numbers, IBAN codes) |
+
+## Considerations before use
+
+- You must follow [GDPR guidance](https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/the-research-provisions/principles-and-grounds-for-processing/) when processing personal data using this package.
+- This package has been designed as a *first pass* for standardised personal data removal. 
+- Users are encouraged to check and confirm outputs and conduct manual reviews where necessary, e.g. when cleaning high risk datasets.
+- It is up to the user to assess whether this removal process needs to be supplemented by other methods for their given dataset and security requirements.
+
+### Input data
+
+- This package is designed for text-based documents structured as a list of strings.
+- It performs best when contextual meaning can be inferred from the text.
+- For best results, input text should therefore resemble natural language. 
+- **Highly fragmented, informal, technical, or syntactically broken text may reduce detection accuracy and lead to incomplete or incorrect name detection.**
+
+### Biases and evaluation
+
+- `idscrub` supports integration with SpaCy and Hugging Face models for name cleaning.
+- These models are state-of-the-art, capable of identifying approximately 90% of named entities, but **may not remove all names**.
+- **Biases present in these models due to their training data may affect performance**. For example:
+    - English names may be more reliably identified than names common in other languages.
+    - Uncommon or non-Western naming conventions may be missed or misclassified.
+
+> [!IMPORTANT]
+> * See [our wiki](https://github.com/uktrade/idscrub/wiki/Evaluation) for further details and notes on our evaluation of `idscrub`.
+
+### Models
+
+* Only Spacy's `en_core_web_trf` and no Hugging Face models have been formally evaluated.
+* We therefore recommend that the current default `en_core_web_trf` is used for name scrubbing. **Other models need to be evaluated by the user.**
+
+## Similar Python packages
+
+* Similar packages exist for undertaking this task, such as [Presidio](https://microsoft.github.io/presidio/), [Scrubadub](https://github.com/LeapBeyond/scrubadub) and [Sanityze](https://github.com/UBC-MDS/sanityze). 
+* Development of `idscrub` was undertaken to: 
+
+    * Bring together different scrubbing methods across the Department for Business and Trade.
+    * Adhere to infrastructure requirements.
+    * Guarantee future stability and maintainability.
+    * Encourage future scrubbing methods to be added collaboratively and transparently.
+    * Allow for full flexibility depending on the use case and required outputs.
+    
+* To leverage the power of other packages, we have added methods that allow you to interact with them. These include: `IDScrub.presidio()` and `IDScrub.google_phone_numbers()`. See the [usage example notebook](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) and method docstrings for further information.
+
+## AI declaration
+
+AI has been used in the development of `idscrub`, primarily to develop regular expressions, suggest code refinements and draft documentation.
+
+## Development setup
+
+This project is managed by [uv](https://docs.astral.sh/uv/).
+
+To install all dependencies for this project, run:
+
+```console
+uv sync --all-extras
+```
+
+If you do not have Python 3.12, run:
+
+```console
+uv python install 3.12
+```
+
+To run tests:
+
+```console
+uv run pytest
+```
+
+or
+
+```console
+make test
+```
+
+## Author 
+
+Analytical Data Science, Department for Business and Trade

idscrub-1.0.1/SECURITY_CHECKLIST.md

@@ -0,0 +1,117 @@
+# Security Checklist
+
+This checklist is designed to make it easier to improve the security posture of a GitHub repository.
+
+- It is mandatory for public repositories.
+- This checklist must be copied over to the root of the repository.
+- The repository steward is responsible for populating the checklist, or at least approving the related pull request.
+- Any feedback should be shared with the GitHub Security working group.
+
+## Checklist
+
+- [x] [Setup the pre-commit hook framework](#setup-the-pre-commit-hook-framework)
+- [x] [Setup custom properties on the repository](#setup-custom-properties-on-the-repository)
+- [x] [Apply the correct github security policy](#apply-the-correct-github-security-policy)
+- [x] [Ensure CODEOWNERS file exists](#ensure-codeowners-file-exists)
+- [x] [Copy the SECURITY_CHECKLIST.md file](#copy-the-security_checklistmd-file)
+- [x] [Review the GitHub CI/CD overview](#review-the-github-cicd-overview)
+- [x] [Review the GitHub Safety Tips](#review-github-safety-tips)
+- [x] [Add Steward to Repository access](#add-at-least-one-steward-to-repository-access)
+- [x] [Review and limit maintainers with admin rights to the strict minimum](#review-and-limit-maintainers-with-admin-rights-to-the-strict-minimum)
+- [x] [Review the Pull Request template](#review-pull-request-template)
+- [x] [Review the SECURITY.md policy](#review-securitymd-policy)
+
+## Setup the pre-commit hook framework
+
+Several uktrade repositories already make use of the pre-commit framework for flagging code quality issues before pushing. Even in the repositories that have the pre-commit framework installed, it is still optional for an individual engineer to either avoid configuring the commit hooks, or skipping them entirely using the `--no-verify` cli argument.
+
+As part of the go live process, each engineer making changes to the repository being reopened will be required to install the organisation wide pre-commit hooks locally. When a PR is opened, an organisation level github action will run to confirm the pre-commit hooks ran on the engineers machine and will block any PRs that have not run these hooks.
+
+Instructions have been added to the [dbt hooks repository](https://github.com/uktrade/github-standards/blob/main/README.md#usage) to provide guidance on adding these organisation wide pre-commit hooks to an individual repository
+
+## Setup custom properties on the repository
+
+A set of github tags have been created at an organisation level, these must be applied to a repository to allow organisation level github actions to run on each pull request.
+
+### Mandatory custom properties
+
+- `reusable_workflow_opt_in`: This one has to be applied and set to `true` to allow this repository to apply the correct organisation branch protection ruleset and run the necessary github workflows on each PR
+- `ddat_portfolio`: The portfolio inside DDAT this repository belongs to. If your portfolio is missing, this can be added by raising an SRE ticket.
+
+### Optional custom properties
+
+- `is_docker`: If this repository builds a docker image, this tag should be added to run docker related github workflows
+- `language`: All languages used by this repository should be selected, and github workflows will run with dedicated checks on that language.
+
+## Apply the correct github security policy
+
+To add the new security policy, follow these instructions:
+
+1. As an organisation administrator, navigate to the [security config page](https://github.com/organizations/uktrade/settings/security_products).
+1. Scroll down to the **Apply configurations** sections, and enter the name of the repository to be made public in the filter input field
+1. Use the checkbox next to the results list to select all repositories being made public, then use the **Apply configuration** button to select the **Default DBT security** configuration
+1. A confirmation modal will appear displaying a summary of the action being made. Click the apply button
+1. In the repository that has had the new policy applied, navigate to the **Advanced Security** page in the repository settings. At the top of the page there should be a banner message **Modifications to some settings have been blocked by organization administrators.**
+
+### Optional: Setup CodeQL to allow PRs from repository forks
+
+For most repositories, the default CodeQL configuration applied by the **Default DBT security** policy will be sufficient. However, this default configuration does not currently support scanning PRs raised from a fork of a repository. If your repository needs to accept PRs from a fork, you must follow these steps to switch to the advanced CodeQL setup:
+
+1. Open the GitHub settings page, and navigate to the Advanced Security section using the left hand menu
+1. Scroll down to the Code Scanning section, under the Tools sub-section there will be an item for CodeQL analysis
+1. Click the ... button next to Default setup text, then choose the Switch to advanced option from the menu
+1. On the popup, click the Disable CodeQL button. Although you are disabling CodeQL, there is still a branch protection rule in place that blocks a PR unless a CodeQL scan is detected. Disabling here will not allow PRs to be merged
+1. The GitHub online editor will open to create a new file called codeql.yml in your repo, and the contents of this file will be prefilled with the languages CodeQL has detected in your repo. You can modify the contents of this file if needed, however you must leave the workflow name as `CodeQL Advanced`
+1. Once happy with the workflow file contents, click the green Commit changes button to trigger a PR to merge this into the main branch
+1. Approve and merge the PR with this workflow file. Once merged, the CodeQL scan will perform an initial scan that can take a while but you can track the progress by viewing the Actions tab for your repository
+
+## Ensure CODEOWNERS file exists
+
+The organisation rulesets require a CODEOWNERS file to be present in the repository. If you don't already have one of these, github has produced [documentation explaining](https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners) what they are and why they are used.
+
+## Copy the SECURITY_CHECKLIST.md file
+
+To allow tracking of repositories that have successfully completed the reopening process, this file must be copied to the root of your repository and each of the items in the Checklist marked as completed
+
+## Review the GitHub CI/CD overview
+
+Internal contributors to the repository should review the CI/CD overview below
+![CI/CD overview](assets/CI-CD%20pipeline.svg)
+
+## Review GitHub Safety Tips
+
+Internal contributors to the repository should review the [GitHub Safety Tips](https://uktrade.atlassian.net/wiki/x/n4AEKQE)
+
+## Add at least one steward to repository access
+
+To ensure correct governance of a repository, at least one steward must be added. This will usually be the most senior engineer on the team. To add a steward to a repository:
+
+1. Open the `Collaborators and teams` settings page. The url for this is `https://github.com/uktrade/REPO_NAME/github-standards/settings/access`
+1. Use the `Add people` button to open the people finder autocomplete box.
+1. Find and click the user who is going to be a steward
+1. On the Choose a role page, select the `Steward` role.
+1. Repeat for any additional users who are going to be a steward
+
+## Review and limit maintainers with admin rights to the strict minimum
+
+You should review who has been assigned the github `admin` role. The `write` role is sufficient to allow team members to commit changes and raise pull requests
+
+## Review Pull Request template
+
+If your repository does not already contain a pull_request_template.md file, by default you will inherit the template from this repository. If you are already using your own template, you should add this section to remind reviewers they should be ensuring no secret values are visible
+
+```
+## Reviewer Checklist
+
+- [ ] I have reviewed the PR and ensured no secret values are present
+```
+
+## Review SECURITY.md policy
+
+This repository contain the SECURITY.md file, which is inherited by all repositories in the uktrade organisation account. This file should be read and understood by the repository steward, and discussed with the team to ensure all engineers understand the tooling that has been put in place
+
+## More information
+
+For more information about GitHub security standards, please refer [to this link](https://dbis.sharepoint.com/:w:/r/sites/DDaTDirectorate/Shared%20Documents/Work%20-%20GitHub%20Security/Github%20Security%20Framework/Guidelines%20and%20Policies/GitHub%20Security%20Standards%20v0.5.docx?d=wb29cd9b99ca042deb5c0cd8d670966d9&csf=1&web=1&e=6ITbnL)
+
+For more details about the security features please refer to the [GitHub Standards](https://github.com/uktrade/github-standards) repo.

idscrub-1.0.1/idscrub/__init__.py

@@ -0,0 +1 @@
+from .scrub import IDScrub as IDScrub

idscrub-1.0.1/idscrub/locations.py

@@ -0,0 +1,10 @@
+import os
+from pathlib import Path
+
+PROJECT_DIR = Path(__file__).resolve().parents[1]
+
+DOWNLOAD_DIR = Path.cwd()
+
+DATA_HOME = os.path.join(PROJECT_DIR, "data")
+
+NOTEBOOKS_HOME = os.path.join(PROJECT_DIR, "notebooks")