idscrub 0.1.0__tar.gz

idscrub-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,89 @@
+name: CI
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: test 
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    strategy:
+      matrix:
+        python-version:
+          - "3.12"
+          - "3.13"
+
+    steps:
+      - uses: actions/checkout@v5
+
+      - name: Install uv and Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          enable-cache: true
+          cache-dependency-glob: "uv.lock"
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install the project
+        run: uv sync --frozen --all-extras --dev
+
+      - name: Run Python tests
+        run: uv run pytest
+
+      - name: Dump docker logs
+        if: failure()
+        uses: jwalton/gh-docker-logs@v2
+
+  scan-for-secrets:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Run TruffleHog scan
+        run: |
+          docker run --rm -v ${{ github.workspace }}:/repo \
+            trufflesecurity/trufflehog:latest \
+            git file:///repo \
+            --since-commit main \
+            --branch ${{ github.head_ref || github.ref_name }} \
+            --github-actions \
+            --fail
+
+  build-package:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+
+    steps:
+      - uses: actions/checkout@v5
+        with:
+          fetch-depth: 0
+
+      - name: Install uv and Python
+        uses: astral-sh/setup-uv@v6
+        with:
+          python-version: "3.12"
+
+      - name: Build package
+        run: uv build
+
+  ci-success:
+    needs: [test, scan-for-secrets, build-package]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    
+    steps:
+      - name: CI success
+        run: echo "All CI checks passed!"

idscrub-0.1.0/.gitignore

@@ -0,0 +1,18 @@
+# Python-generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# Virtual environments
+.venv
+
+# idscrub specific
+huggingface
+
+.pytest_cache
+.ruff_cache
+.coverage
+.DS_Store

idscrub-0.1.0/.pre-commit-config.yaml

@@ -0,0 +1,22 @@
+repos:
+- repo: local
+  hooks:
+    # Run the formatter.
+    - id: ruff
+      name: ruff
+      description: "Lint and format code using Ruff"
+      entry: make format
+      language: system
+      types_or: [python, pyi]
+      pass_filenames: false
+
+# Mandatory internal hooks
+- repo: https://github.com/uktrade/github-standards
+  rev: v1.1.2  # update periodically with pre-commit autoupdate
+  hooks:
+    - id: run-security-scan
+      verbose: false
+    - id: run-personal-data-scan
+      verbose: false
+    - id: validate-security-scan
+      verbose: false

idscrub-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Department for Business and Trade
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

idscrub-0.1.0/Makefile

@@ -0,0 +1,15 @@
+.PHONY: clean precommit test 
+
+## Delete all compiled Python files
+clean:
+	find . -type f -name "*.py[co]" -delete
+	find . -type d -name "__pycache__" -delete
+
+## Reformat, lint
+format:
+	uv run ruff format .
+	uv run ruff check . --fix
+
+## Run Python tests
+test:
+	uv run pytest

idscrub-0.1.0/PKG-INFO

@@ -0,0 +1,147 @@
+Metadata-Version: 2.4
+Name: idscrub
+Version: 0.1.0
+Author: Department for Business and Trade
+Requires-Python: >=3.12
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: ipykernel>=7.1.0
+Requires-Dist: numpy>=2.3.4
+Requires-Dist: pandas>=2.3.3
+Requires-Dist: phonenumbers>=9.0.18
+Requires-Dist: pip>=25.3
+Requires-Dist: spacy-transformers>=1.3.9
+Requires-Dist: tqdm>=4.67.1
+Requires-Dist: presidio-analyzer
+Requires-Dist: presidio-anonymizer
+Provides-Extra: trf
+Requires-Dist: en_core_web_trf; extra == "trf"
+Dynamic: license-file
+
+# idscrub 🧽✨
+
+## Project Info
+
+* This package removes (*✨scrubs✨*) identifying personal data from text using [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) and [named-entity recognition](https://en.wikipedia.org/wiki/Named-entity_recognition).
+
+> [!WARNING]
+> You must follow [GDPR guidance](https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/the-research-provisions/principles-and-grounds-for-processing/) when processing personal data using this package.
+>
+> Specifically, you must:
+>
+> - **Update privacy notices**: Clearly state this processing activity in new or existing privacy notices before using the package.
+> - **Ensure secure deletion**: Remove any temporary or intermediary files and outputs in a secure manner.
+> - **Ensure data subject rights upheld**: Ensure individuals can access, correct, or erase their data as required.
+> - **Maintain processing records**: Document how personal data is handled and for what purpose.
+
+### Description
+
+* Names and other personally identifying information are often present in text.
+* This information may need to be removed prior to further analysis in many cases.
+* `idscrub` provides a standardised way to do this in the Department for Business and Trade. 
+
+### Expected Outputs
+
+* A list of text with names and other identifying information removed.
+
+> [!WARNING]
+> * This package has been designed as a *first pass* for standardised personal data removal. 
+> * Users are encouraged to check and confirm outputs and conduct manual reviews where necessary, e.g. when cleaning high risk datasets.
+> * It is up to the user to assess whether this removal process needs to be supplemented by other methods for their given dataset and security requirements.
+
+### Data
+
+- This package is designed for text-based documents structured as a list of strings.
+- It performs best when contextual meaning can be inferred from the text.
+- For best results, input text should therefore resemble natural language. 
+- **Highly fragmented, informal, technical, or syntactically broken text may reduce detection accuracy and lead to incomplete or incorrect name detection.**
+
+### Biases and evaluation
+
+- `idscrub` supports integration with SpaCy and Hugging Face models for name cleaning.
+- These models are state-of-the-art, capable of identifying approximately 90% of named entities, but **may not remove all names**.
+- **Biases present in these models due to their training data may affect performance**. For example:
+    - English names may be more reliably identified than names common in other languages.
+    - Uncommon or non-Western naming conventions may be missed or misclassified.
+
+> [!IMPORTANT]
+> * See [our wiki](https://github.com/uktrade/idscrub/wiki/Evaluation) for further details and notes on our evaluation of `idscrub`.
+
+### Models and Memory
+
+* Only Spacy's `en_core_web_trf` and no Hugging Face models have been formally evaluated.
+* We therefore recommend that the current default `en_core_web_trf` is used for name scrubbing. **Other models need to be evaluated by the user.**
+
+> [!IMPORTANT]
+> Spacy and Hugging Face models have high memory requirements. To avoid memory-related errors. Clear the auto-generated `huggingface` folder if not in use. Do not push the `huggingface` folder (or user-defined equivalent) to GitHub.
+
+## Similar Python packages
+
+* Similar packages exist for undertaking this task, such as [presidio](https://microsoft.github.io/presidio/), [scrubadub](https://github.com/LeapBeyond/scrubadub) and [sanityze](https://github.com/UBC-MDS/sanityze). 
+* Development of `idscrub` was undertaken to: bring together different scrubbing methods across the department, adhere to infrastructure requirements, guarantee future stability and maintainability, and encourage future scrubbing methods to be added collaboratively and transparently. 
+* To leverage the power of other packages, we have added methods that allow you to interact with them. These include: `IDScrub.presidio()` and `IDScrub.google_phone_numbers()`. See the [usage example notebook](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) and method docstrings for further information.
+
+
+## Installation
+
+`idscrub` can be installed using `pip` into a Python **>=3.12** environment. Example (with spaCy model installed):
+
+```console
+pip install 'git+ssh://git@github.com/uktrade/idscrub.git#egg=idscrub[trf]'
+```
+or without spaCy installed (it will be installed automatically if name cleaning methods are called):
+
+```console
+pip install 'git+ssh://git@github.com/uktrade/idscrub.git'
+```
+
+## How to use the code
+
+Basic usage example (see `notebooks/basic_usage.ipynb` for further examples):
+
+```python
+from idscrub import IDScrub
+
+scrub = IDScrub(['Our names are Hamish McDonald, L. Salah, and Elena Suárez.', 'My number is +441111111111 and I live at AA11 1AA, Lapland.'])
+scrubbed_texts = scrub.all()
+
+print(scrubbed_texts)
+
+# Output: ['Our names are [PERSON], [PERSON], and [PERSON].', 'My number is [PHONENO] and I live at [POSTCODE], [LOCATION].']
+```
+
+## AI Declaration
+
+AI has been used in the development of `idscrub`, primarily to develop regular expressions, suggest code refinements and draft documentation.
+
+## Development setup
+
+This project is managed by [uv](https://docs.astral.sh/uv/).
+
+To install all dependencies for this project, run:
+
+```console
+uv sync --all-extras
+```
+
+If you do not have Python 3.12, run:
+
+```console
+uv python install 3.12
+```
+
+To run tests:
+
+```console
+uv run pytest
+```
+
+or
+
+```console
+make test
+```
+
+## Author 
+
+Analytical Data Science, Department for Business and Trade

idscrub-0.1.0/README.md

@@ -0,0 +1,127 @@
+# idscrub 🧽✨
+
+## Project Info
+
+* This package removes (*✨scrubs✨*) identifying personal data from text using [regular expressions](https://en.wikipedia.org/wiki/Regular_expression) and [named-entity recognition](https://en.wikipedia.org/wiki/Named-entity_recognition).
+
+> [!WARNING]
+> You must follow [GDPR guidance](https://ico.org.uk/for-organisations/uk-gdpr-guidance-and-resources/the-research-provisions/principles-and-grounds-for-processing/) when processing personal data using this package.
+>
+> Specifically, you must:
+>
+> - **Update privacy notices**: Clearly state this processing activity in new or existing privacy notices before using the package.
+> - **Ensure secure deletion**: Remove any temporary or intermediary files and outputs in a secure manner.
+> - **Ensure data subject rights upheld**: Ensure individuals can access, correct, or erase their data as required.
+> - **Maintain processing records**: Document how personal data is handled and for what purpose.
+
+### Description
+
+* Names and other personally identifying information are often present in text.
+* This information may need to be removed prior to further analysis in many cases.
+* `idscrub` provides a standardised way to do this in the Department for Business and Trade. 
+
+### Expected Outputs
+
+* A list of text with names and other identifying information removed.
+
+> [!WARNING]
+> * This package has been designed as a *first pass* for standardised personal data removal. 
+> * Users are encouraged to check and confirm outputs and conduct manual reviews where necessary, e.g. when cleaning high risk datasets.
+> * It is up to the user to assess whether this removal process needs to be supplemented by other methods for their given dataset and security requirements.
+
+### Data
+
+- This package is designed for text-based documents structured as a list of strings.
+- It performs best when contextual meaning can be inferred from the text.
+- For best results, input text should therefore resemble natural language. 
+- **Highly fragmented, informal, technical, or syntactically broken text may reduce detection accuracy and lead to incomplete or incorrect name detection.**
+
+### Biases and evaluation
+
+- `idscrub` supports integration with SpaCy and Hugging Face models for name cleaning.
+- These models are state-of-the-art, capable of identifying approximately 90% of named entities, but **may not remove all names**.
+- **Biases present in these models due to their training data may affect performance**. For example:
+    - English names may be more reliably identified than names common in other languages.
+    - Uncommon or non-Western naming conventions may be missed or misclassified.
+
+> [!IMPORTANT]
+> * See [our wiki](https://github.com/uktrade/idscrub/wiki/Evaluation) for further details and notes on our evaluation of `idscrub`.
+
+### Models and Memory
+
+* Only Spacy's `en_core_web_trf` and no Hugging Face models have been formally evaluated.
+* We therefore recommend that the current default `en_core_web_trf` is used for name scrubbing. **Other models need to be evaluated by the user.**
+
+> [!IMPORTANT]
+> Spacy and Hugging Face models have high memory requirements. To avoid memory-related errors. Clear the auto-generated `huggingface` folder if not in use. Do not push the `huggingface` folder (or user-defined equivalent) to GitHub.
+
+## Similar Python packages
+
+* Similar packages exist for undertaking this task, such as [presidio](https://microsoft.github.io/presidio/), [scrubadub](https://github.com/LeapBeyond/scrubadub) and [sanityze](https://github.com/UBC-MDS/sanityze). 
+* Development of `idscrub` was undertaken to: bring together different scrubbing methods across the department, adhere to infrastructure requirements, guarantee future stability and maintainability, and encourage future scrubbing methods to be added collaboratively and transparently. 
+* To leverage the power of other packages, we have added methods that allow you to interact with them. These include: `IDScrub.presidio()` and `IDScrub.google_phone_numbers()`. See the [usage example notebook](https://github.com/uktrade/idscrub/blob/main/notebooks/basic_usage.ipynb) and method docstrings for further information.
+
+
+## Installation
+
+`idscrub` can be installed using `pip` into a Python **>=3.12** environment. Example (with spaCy model installed):
+
+```console
+pip install 'git+ssh://git@github.com/uktrade/idscrub.git#egg=idscrub[trf]'
+```
+or without spaCy installed (it will be installed automatically if name cleaning methods are called):
+
+```console
+pip install 'git+ssh://git@github.com/uktrade/idscrub.git'
+```
+
+## How to use the code
+
+Basic usage example (see `notebooks/basic_usage.ipynb` for further examples):
+
+```python
+from idscrub import IDScrub
+
+scrub = IDScrub(['Our names are Hamish McDonald, L. Salah, and Elena Suárez.', 'My number is +441111111111 and I live at AA11 1AA, Lapland.'])
+scrubbed_texts = scrub.all()
+
+print(scrubbed_texts)
+
+# Output: ['Our names are [PERSON], [PERSON], and [PERSON].', 'My number is [PHONENO] and I live at [POSTCODE], [LOCATION].']
+```
+
+## AI Declaration
+
+AI has been used in the development of `idscrub`, primarily to develop regular expressions, suggest code refinements and draft documentation.
+
+## Development setup
+
+This project is managed by [uv](https://docs.astral.sh/uv/).
+
+To install all dependencies for this project, run:
+
+```console
+uv sync --all-extras
+```
+
+If you do not have Python 3.12, run:
+
+```console
+uv python install 3.12
+```
+
+To run tests:
+
+```console
+uv run pytest
+```
+
+or
+
+```console
+make test
+```
+
+## Author 
+
+Analytical Data Science, Department for Business and Trade

idscrub-0.1.0/idscrub/__init__.py

@@ -0,0 +1 @@
+from .scrub import IDScrub as IDScrub

idscrub-0.1.0/idscrub/locations.py

@@ -0,0 +1,10 @@
+import os
+from pathlib import Path
+
+PROJECT_DIR = Path(__file__).resolve().parents[1]
+
+DOWNLOAD_DIR = Path.cwd()
+
+DATA_HOME = os.path.join(PROJECT_DIR, "data")
+
+NOTEBOOKS_HOME = os.path.join(PROJECT_DIR, "notebooks")