data-sampler 3.2.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data_sampler/__init__.py +82 -0
- data_sampler/__main__.py +7 -0
- data_sampler/_logging.py +61 -0
- data_sampler/_names.py +60 -0
- data_sampler/anonymize.py +463 -0
- data_sampler/cli.py +363 -0
- data_sampler/engine.py +628 -0
- data_sampler/io.py +75 -0
- data_sampler/report.py +289 -0
- data_sampler/sampling.py +193 -0
- data_sampler/stats.py +174 -0
- data_sampler/tui/__init__.py +16 -0
- data_sampler/tui/app.py +904 -0
- data_sampler/workflow.py +285 -0
- data_sampler-3.2.1.dist-info/METADATA +427 -0
- data_sampler-3.2.1.dist-info/RECORD +19 -0
- data_sampler-3.2.1.dist-info/WHEEL +4 -0
- data_sampler-3.2.1.dist-info/entry_points.txt +3 -0
- data_sampler-3.2.1.dist-info/licenses/LICENSE +21 -0
data_sampler/__init__.py
ADDED
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""data-sampler: representative sampling and anonymization for tabular data.
|
|
2
|
+
|
|
3
|
+
Public API::
|
|
4
|
+
|
|
5
|
+
import data_sampler as ds
|
|
6
|
+
|
|
7
|
+
df = ds.load_file("data.csv")
|
|
8
|
+
stats = ds.compute_stats(df) # Data Wrangler-style column stats
|
|
9
|
+
result = ds.sample(df, 500, exclude_columns=["notes"])
|
|
10
|
+
print(ds.format_stratification_report(df, result))
|
|
11
|
+
anon = ds.anonymize(result.data, {"name": "names", "id": ("sequential_id", {"start": 1000})})
|
|
12
|
+
ds.save_output(anon, "data.csv", tag="sample_500_anon")
|
|
13
|
+
|
|
14
|
+
ds.run_tui() # launch the terminal UI
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
from __future__ import annotations
|
|
18
|
+
|
|
19
|
+
__version__ = "3.2.1"
|
|
20
|
+
|
|
21
|
+
from .io import SUPPORTED_EXTENSIONS, list_sheets, load_file, save_output
|
|
22
|
+
from .report import (
|
|
23
|
+
column_histogram_data,
|
|
24
|
+
format_column_histograms,
|
|
25
|
+
format_distribution,
|
|
26
|
+
format_stratification_report,
|
|
27
|
+
)
|
|
28
|
+
from .sampling import (
|
|
29
|
+
SampleResult,
|
|
30
|
+
find_stratification_columns,
|
|
31
|
+
sample,
|
|
32
|
+
stratified_sample,
|
|
33
|
+
)
|
|
34
|
+
from .stats import ColumnStats, compute_column_stats, compute_stats, sparkline
|
|
35
|
+
|
|
36
|
+
__all__ = [
|
|
37
|
+
"__version__",
|
|
38
|
+
"SUPPORTED_EXTENSIONS",
|
|
39
|
+
"load_file",
|
|
40
|
+
"list_sheets",
|
|
41
|
+
"save_output",
|
|
42
|
+
"ColumnStats",
|
|
43
|
+
"compute_stats",
|
|
44
|
+
"compute_column_stats",
|
|
45
|
+
"sparkline",
|
|
46
|
+
"SampleResult",
|
|
47
|
+
"sample",
|
|
48
|
+
"stratified_sample",
|
|
49
|
+
"find_stratification_columns",
|
|
50
|
+
"format_stratification_report",
|
|
51
|
+
"format_distribution",
|
|
52
|
+
"column_histogram_data",
|
|
53
|
+
"format_column_histograms",
|
|
54
|
+
"anonymize",
|
|
55
|
+
"make_anonymizer",
|
|
56
|
+
"AnonymizationPlan",
|
|
57
|
+
"suggest_type",
|
|
58
|
+
"TYPE_OPTIONS",
|
|
59
|
+
"run_tui",
|
|
60
|
+
]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def __getattr__(name: str):
|
|
64
|
+
# Lazy imports: keep pandas-only workflows fast and let the core API work
|
|
65
|
+
# even while optional pieces (TUI) are not yet importable.
|
|
66
|
+
if name in ("anonymize", "make_anonymizer"):
|
|
67
|
+
# importlib, not `from . import anonymize`: the fromlist form calls
|
|
68
|
+
# hasattr(package, "anonymize"), which re-enters this __getattr__.
|
|
69
|
+
import importlib
|
|
70
|
+
|
|
71
|
+
_anon = importlib.import_module(".anonymize", __name__)
|
|
72
|
+
return getattr(_anon, name)
|
|
73
|
+
if name in ("AnonymizationPlan", "suggest_type", "TYPE_OPTIONS"):
|
|
74
|
+
import importlib
|
|
75
|
+
|
|
76
|
+
_wf = importlib.import_module(".workflow", __name__)
|
|
77
|
+
return getattr(_wf, name)
|
|
78
|
+
if name == "run_tui":
|
|
79
|
+
from .tui import run_tui
|
|
80
|
+
|
|
81
|
+
return run_tui
|
|
82
|
+
raise AttributeError(f"module 'data_sampler' has no attribute {name!r}")
|
data_sampler/__main__.py
ADDED
data_sampler/_logging.py
ADDED
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Central logging setup, controlled by the DATA_SAMPLER_LOG env var.
|
|
2
|
+
|
|
3
|
+
Levels: ``quiet`` (warnings only), ``info``, ``verbose`` (debug).
|
|
4
|
+
Defaults to ``verbose`` during development. Set ``DATA_SAMPLER_LOG=quiet``
|
|
5
|
+
to silence, or ``DATA_SAMPLER_LOG_FILE=<path>`` to divert output to a file
|
|
6
|
+
(the TUI does this automatically so log lines cannot corrupt the display).
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import logging
|
|
12
|
+
import os
|
|
13
|
+
import sys
|
|
14
|
+
|
|
15
|
+
_LEVELS = {
|
|
16
|
+
"quiet": logging.WARNING,
|
|
17
|
+
"info": logging.INFO,
|
|
18
|
+
"verbose": logging.DEBUG,
|
|
19
|
+
"debug": logging.DEBUG,
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
_configured = False
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _configure() -> None:
|
|
26
|
+
global _configured
|
|
27
|
+
if _configured:
|
|
28
|
+
return
|
|
29
|
+
root = logging.getLogger("data_sampler")
|
|
30
|
+
level_name = os.environ.get("DATA_SAMPLER_LOG", "verbose").lower()
|
|
31
|
+
root.setLevel(_LEVELS.get(level_name, logging.DEBUG))
|
|
32
|
+
|
|
33
|
+
log_file = os.environ.get("DATA_SAMPLER_LOG_FILE")
|
|
34
|
+
handler: logging.Handler
|
|
35
|
+
if log_file:
|
|
36
|
+
handler = logging.FileHandler(log_file, encoding="utf-8")
|
|
37
|
+
else:
|
|
38
|
+
handler = logging.StreamHandler(sys.stderr)
|
|
39
|
+
handler.setFormatter(
|
|
40
|
+
logging.Formatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s")
|
|
41
|
+
)
|
|
42
|
+
root.addHandler(handler)
|
|
43
|
+
_configured = True
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def get_logger(name: str) -> logging.Logger:
|
|
47
|
+
"""Return a package logger; configures handlers on first use."""
|
|
48
|
+
_configure()
|
|
49
|
+
return logging.getLogger(name)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def redirect_to_file(path: str) -> None:
|
|
53
|
+
"""Swap all package log handlers for a file handler (used by the TUI)."""
|
|
54
|
+
root = logging.getLogger("data_sampler")
|
|
55
|
+
for h in list(root.handlers):
|
|
56
|
+
root.removeHandler(h)
|
|
57
|
+
handler = logging.FileHandler(path, encoding="utf-8")
|
|
58
|
+
handler.setFormatter(
|
|
59
|
+
logging.Formatter("%(asctime)s %(levelname)-7s %(name)s: %(message)s")
|
|
60
|
+
)
|
|
61
|
+
root.addHandler(handler)
|
data_sampler/_names.py
ADDED
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
'''Bundled name library for the "names" anonymizer.'''
|
|
2
|
+
|
|
3
|
+
FIRST_NAMES = (
|
|
4
|
+
"Aarav", "Aditi", "Adriana", "Ahmed", "Aisha", "Akira", "Alejandro", "Alexandra", "Alice", "Amara",
|
|
5
|
+
"Amelia", "Amina", "Ananya", "Anders", "Andrea", "Angela", "Anika", "Anita", "Anthony", "Antonio",
|
|
6
|
+
"Arjun", "Arthur", "Astrid", "Ava", "Ayesha", "Beatriz", "Benjamin", "Bianca", "Bilal", "Brian",
|
|
7
|
+
"Bruno", "Camila", "Carlos", "Carmen", "Caroline", "Catherine", "Chiara", "Chloe", "Christopher", "Clara",
|
|
8
|
+
"Daniel", "David", "Deepa", "Diana", "Diego", "Dmitri", "Dylan", "Elena", "Elias", "Elif",
|
|
9
|
+
"Emeka", "Emily", "Emma", "Enrique", "Eric", "Esther", "Ethan", "Fatima", "Felipe", "Felix",
|
|
10
|
+
"Fiona", "Francesca", "Gabriel", "Grace", "Gustavo", "Hana", "Hannah", "Haruto", "Hassan", "Helena",
|
|
11
|
+
"Henry", "Hiroshi", "Hugo", "Ibrahim", "Imani", "Ines", "Irene", "Isaac", "Isabella", "Ishaan",
|
|
12
|
+
"Ivan", "Jack", "Jacob", "James", "Jasmine", "Javier", "Jin", "Joanna", "John", "Jonas",
|
|
13
|
+
"Jorge", "Jose", "Julia", "Julien", "Kavya", "Keiko", "Kemal", "Kenji", "Kiara", "Kwame",
|
|
14
|
+
"Laila", "Lars", "Laura", "Leah", "Leila", "Leo", "Leonardo", "Liam", "Lin", "Linda",
|
|
15
|
+
"Lucas", "Lucia", "Luis", "Luna", "Magnus", "Maja", "Malik", "Marcus", "Margaret", "Maria",
|
|
16
|
+
"Mariam", "Marina", "Mario", "Marta", "Martin", "Mateo", "Matteo", "Maya", "Mei", "Michael",
|
|
17
|
+
"Miguel", "Mina", "Ming", "Mohammed", "Monica", "Naledi", "Naomi", "Natalia", "Nathan", "Nia",
|
|
18
|
+
"Nikhil", "Nikolai", "Nina", "Noah", "Noor", "Nora", "Olga", "Oliver", "Olivia", "Omar",
|
|
19
|
+
"Oscar", "Pablo", "Paola", "Patricia", "Paulo", "Pedro", "Peter", "Petra", "Priya", "Rachel",
|
|
20
|
+
"Rafael", "Rahul", "Rania", "Ravi", "Rebecca", "Renata", "Rhea", "Ricardo", "Robert", "Rohan",
|
|
21
|
+
"Rosa", "Ruby", "Ryan", "Saanvi", "Salma", "Samuel", "Sana", "Santiago", "Sara", "Sean",
|
|
22
|
+
"Sergei", "Simone", "Sofia", "Sonia", "Sophie", "Stefan", "Susana", "Tara", "Tariq", "Thomas",
|
|
23
|
+
"Tomas", "Uma", "Valentina", "Victor", "Vikram", "Wei", "William", "Yuki", "Zara", "Zoe",
|
|
24
|
+
)
|
|
25
|
+
|
|
26
|
+
MIDDLE_NAMES = (
|
|
27
|
+
"Ann", "Bea", "Belle", "Blair", "Blake", "Brooke", "Claire", "Cole", "Dale", "Dawn",
|
|
28
|
+
"Dean", "Devi", "Drew", "Elise", "Elle", "Eve", "Faith", "Finn", "Gail", "Glen",
|
|
29
|
+
"Grant", "Gray", "Gwen", "Hope", "Jade", "Jane", "Jay", "Jean", "Joan", "Jude",
|
|
30
|
+
"June", "Kai", "Kate", "Kay", "Kit", "Lane", "Lark", "Lee", "Luz", "Mae",
|
|
31
|
+
"Marie", "Max", "May", "Neil", "Nell", "Noel", "Paige", "Pearl", "Quinn", "Rae",
|
|
32
|
+
"Ray", "Reed", "Reese", "Rose", "Sage", "Shea", "Sky", "Tate", "Tess", "Wren",
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
LAST_NAMES = (
|
|
36
|
+
"Acharya", "Adeyemi", "Agarwal", "Aguilar", "Ali", "Alvarez", "Andersen", "Anderson", "Baker", "Banerjee",
|
|
37
|
+
"Barnes", "Becker", "Bell", "Bennett", "Bhat", "Bianchi", "Boateng", "Bose", "Brooks", "Brown",
|
|
38
|
+
"Bryant", "Campbell", "Carter", "Castillo", "Castro", "Chan", "Chandra", "Chatterjee", "Chavez", "Chen",
|
|
39
|
+
"Cho", "Choi", "Chowdhury", "Clark", "Cohen", "Collins", "Conti", "Costa", "Cruz", "Das",
|
|
40
|
+
"Davies", "Davis", "Delgado", "Desai", "Diallo", "Diaz", "Dubois", "Duong", "Dutta", "Edwards",
|
|
41
|
+
"Eriksson", "Esposito", "Evans", "Fernandez", "Ferrari", "Fischer", "Fleming", "Flores", "Foster", "Fujimoto",
|
|
42
|
+
"Garcia", "Ghosh", "Gomez", "Gonzalez", "Gray", "Green", "Griffin", "Gupta", "Haddad", "Hall",
|
|
43
|
+
"Hansen", "Harris", "Hayashi", "Henderson", "Hernandez", "Hill", "Hoffmann", "Howard", "Huang", "Hughes",
|
|
44
|
+
"Hussain", "Ibrahim", "Iqbal", "Ito", "Iyer", "Jackson", "Jain", "Jansen", "Jenkins", "Jensen",
|
|
45
|
+
"Johansson", "Johnson", "Jones", "Joshi", "Kapoor", "Kaur", "Keller", "Khan", "Kim", "King",
|
|
46
|
+
"Kobayashi", "Kowalski", "Krishnan", "Kulkarni", "Kumar", "Larsen", "Laurent", "Lee", "Lewis", "Li",
|
|
47
|
+
"Lim", "Lindberg", "Liu", "Lopez", "Ma", "Malhotra", "Marino", "Martin", "Martinez", "Mbeki",
|
|
48
|
+
"McCarthy", "Mehta", "Mendez", "Mensah", "Meyer", "Miller", "Mishra", "Mitchell", "Mohamed", "Moreau",
|
|
49
|
+
"Morgan", "Mori", "Morris", "Moyo", "Murphy", "Nair", "Nakamura", "Nelson", "Nguyen", "Nielsen",
|
|
50
|
+
"Novak", "Nowak", "Okafor", "Okonkwo", "Oliveira", "Olsen", "Ortega", "Osei", "Park", "Patel",
|
|
51
|
+
"Pereira", "Perez", "Peterson", "Pham", "Phillips", "Pillai", "Popov", "Powell", "Price", "Qureshi",
|
|
52
|
+
"Rahman", "Raman", "Ramirez", "Ramos", "Rao", "Reddy", "Reyes", "Richardson", "Rivera", "Roberts",
|
|
53
|
+
"Robinson", "Rodriguez", "Romano", "Rossi", "Roy", "Russo", "Saito", "Sanchez", "Sanders", "Santos",
|
|
54
|
+
"Sato", "Schmidt", "Schneider", "Scott", "Sen", "Shah", "Sharma", "Silva", "Singh", "Smith",
|
|
55
|
+
"Sokolov", "Srinivasan", "Stewart", "Suzuki", "Takahashi", "Tanaka", "Taylor", "Thompson", "Torres", "Tran",
|
|
56
|
+
"Turner", "Verma", "Walker", "Wang", "Watson", "Weber", "White", "Williams", "Wilson", "Wong",
|
|
57
|
+
"Wright", "Wu", "Yamamoto", "Yang", "Yilmaz", "Yoshida", "Young", "Zhang", "Zhao", "Zhou",
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
ALL = {"first": FIRST_NAMES, "middle": MIDDLE_NAMES, "last": LAST_NAMES}
|
|
@@ -0,0 +1,463 @@
|
|
|
1
|
+
"""Optional per-column anonymizers.
|
|
2
|
+
|
|
3
|
+
Every anonymizer maps each *unique* original value to exactly one replacement
|
|
4
|
+
(consistent mapping), so repeated values stay repeated — the statistical
|
|
5
|
+
variety this project exists to preserve survives anonymization. Missing values
|
|
6
|
+
(NaN/NaT) are left untouched.
|
|
7
|
+
|
|
8
|
+
Two families differ in how completely the distribution survives:
|
|
9
|
+
|
|
10
|
+
- **Relabelling** anonymizers (``names``, ``sequential_id``, ``random_string``,
|
|
11
|
+
``hex``) are *bijective*: distinct originals map to distinct replacements, so
|
|
12
|
+
the exact value-count distribution is preserved (only the labels change).
|
|
13
|
+
- **Jitter** anonymizers (``numeric_jitter``, ``datetime_jitter``) add bounded
|
|
14
|
+
random noise: each replacement stays within ±the configured bound of its
|
|
15
|
+
original, so the range and rough shape survive, but two *nearby* distinct
|
|
16
|
+
values can round/shift onto the same replacement (jitter is noise, not a
|
|
17
|
+
bijection). Widen the bound, add ``round_to``/finer ``unit`` resolution, or
|
|
18
|
+
use a relabelling anonymizer if you need strict distinct-to-distinct output.
|
|
19
|
+
|
|
20
|
+
Available kinds (see :data:`KINDS` for aliases):
|
|
21
|
+
|
|
22
|
+
- ``names`` — replace values with realistic full names drawn from a bundled
|
|
23
|
+
library of first/middle/last names (:class:`NameAnonymizer`).
|
|
24
|
+
- ``sequential_id`` — replace values with ``start, start+interval, ...`` in
|
|
25
|
+
order of first appearance (:class:`SequentialIdAnonymizer`).
|
|
26
|
+
- ``numeric_jitter`` — replace numbers with a random value within ±20 % of
|
|
27
|
+
the original by default (:class:`NumericJitterAnonymizer`).
|
|
28
|
+
- ``datetime_jitter`` — shift each date/time by a random offset within a
|
|
29
|
+
±window (±7 days by default) (:class:`DatetimeJitterAnonymizer`).
|
|
30
|
+
- ``random_string`` / ``hex`` — replace values with random character
|
|
31
|
+
sequences or hexadecimal strings (:class:`RandomStringAnonymizer`).
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
from __future__ import annotations
|
|
35
|
+
|
|
36
|
+
import random
|
|
37
|
+
import string
|
|
38
|
+
from abc import ABC, abstractmethod
|
|
39
|
+
from typing import Any, Hashable, Mapping, Sequence
|
|
40
|
+
|
|
41
|
+
import numpy as np
|
|
42
|
+
import pandas as pd
|
|
43
|
+
|
|
44
|
+
from . import _names
|
|
45
|
+
from ._logging import get_logger
|
|
46
|
+
|
|
47
|
+
log = get_logger(__name__)
|
|
48
|
+
|
|
49
|
+
AnonymizerSpec = "ColumnAnonymizer | str | tuple[str, dict] | dict"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _np_rng(rng: random.Random) -> np.random.Generator:
|
|
53
|
+
"""Derive a numpy Generator from a :class:`random.Random`.
|
|
54
|
+
|
|
55
|
+
Draws 64 bits from ``rng`` so vectorized anonymizers stay deterministic
|
|
56
|
+
for a given seed and column order (``anonymize`` passes one ``Random`` to
|
|
57
|
+
every column in turn), without coupling to numpy's global state.
|
|
58
|
+
"""
|
|
59
|
+
return np.random.default_rng(rng.getrandbits(64))
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
class ColumnAnonymizer(ABC):
|
|
63
|
+
"""Base class: replace each unique original value with one replacement.
|
|
64
|
+
|
|
65
|
+
The pipeline is vectorized. The column is dictionary-encoded once with
|
|
66
|
+
:func:`pandas.factorize` — yielding integer ``codes`` per row plus the
|
|
67
|
+
distinct ``uniques`` in order of first appearance — each subclass produces
|
|
68
|
+
one replacement per unique value via :meth:`build_replacements`, and the
|
|
69
|
+
result is assembled by fancy-indexing (gathering) ``replacements`` with
|
|
70
|
+
``codes``. That is the in-process equivalent of a native join against a
|
|
71
|
+
mapping table: no Python loop over rows and no second hash pass, so cost
|
|
72
|
+
scales with the number of *unique* values plus a vectorized gather over the
|
|
73
|
+
rows. The consistent-mapping guarantee (equal values → equal replacements)
|
|
74
|
+
is inherent to the encoding, and missing values (``NaN``/``NaT``) are
|
|
75
|
+
preserved.
|
|
76
|
+
"""
|
|
77
|
+
|
|
78
|
+
def transform(self, series: pd.Series, rng: random.Random | None = None) -> pd.Series:
|
|
79
|
+
"""Return a copy of ``series`` with all non-null values replaced."""
|
|
80
|
+
rng = rng if rng is not None else random.Random()
|
|
81
|
+
codes, uniques = pd.factorize(series, use_na_sentinel=True)
|
|
82
|
+
if len(uniques) == 0: # all-null (or empty) column: nothing to map
|
|
83
|
+
return series.copy()
|
|
84
|
+
replacements = self.build_replacements(uniques, rng)
|
|
85
|
+
log.debug(
|
|
86
|
+
"%s: column %r — %d unique values mapped over %d rows",
|
|
87
|
+
type(self).__name__, series.name, len(uniques), len(series),
|
|
88
|
+
)
|
|
89
|
+
result = self._gather(series, codes, replacements)
|
|
90
|
+
return self._restore_dtype(series, result)
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _gather(
|
|
94
|
+
series: pd.Series, codes: np.ndarray, replacements: Sequence[Any]
|
|
95
|
+
) -> pd.Series:
|
|
96
|
+
"""Gather ``replacements`` by ``codes``, restoring NaN where code == -1."""
|
|
97
|
+
repl = np.asarray(replacements)
|
|
98
|
+
# a code of -1 marks a missing value; clip it to 0 for the fancy-index
|
|
99
|
+
# (so we never read out of bounds) then mask those positions back to NaN
|
|
100
|
+
safe = np.where(codes >= 0, codes, 0)
|
|
101
|
+
gathered = repl[safe]
|
|
102
|
+
result = pd.Series(gathered, index=series.index, name=series.name)
|
|
103
|
+
if (codes < 0).any():
|
|
104
|
+
result = result.where(pd.Series(codes >= 0, index=series.index))
|
|
105
|
+
return result
|
|
106
|
+
|
|
107
|
+
def build_mapping(
|
|
108
|
+
self, uniques: Sequence[Hashable], rng: random.Random
|
|
109
|
+
) -> dict[Hashable, Any]:
|
|
110
|
+
"""Map each unique original value to its replacement (a dict).
|
|
111
|
+
|
|
112
|
+
Kept for direct callers; :meth:`transform` uses the vectorized
|
|
113
|
+
:meth:`build_replacements` path instead.
|
|
114
|
+
"""
|
|
115
|
+
uniques = list(uniques)
|
|
116
|
+
return dict(zip(uniques, self.build_replacements(uniques, rng)))
|
|
117
|
+
|
|
118
|
+
def _restore_dtype(self, original: pd.Series, result: pd.Series) -> pd.Series:
|
|
119
|
+
# round-trip a pandas nullable string dtype (else its pd.NA marker
|
|
120
|
+
# degrades to a float nan); plain object columns are left as-is
|
|
121
|
+
dt = original.dtype
|
|
122
|
+
if pd.api.types.is_string_dtype(dt) and not pd.api.types.is_object_dtype(dt):
|
|
123
|
+
try:
|
|
124
|
+
return result.astype(dt)
|
|
125
|
+
except (TypeError, ValueError):
|
|
126
|
+
return result
|
|
127
|
+
return result
|
|
128
|
+
|
|
129
|
+
@abstractmethod
|
|
130
|
+
def build_replacements(
|
|
131
|
+
self, uniques: Sequence[Hashable], rng: random.Random
|
|
132
|
+
) -> Sequence[Any]:
|
|
133
|
+
"""Return one replacement per unique value, in the order given.
|
|
134
|
+
|
|
135
|
+
``uniques`` are the column's distinct non-null values in order of first
|
|
136
|
+
appearance (the ``uniques`` array from :func:`pandas.factorize`).
|
|
137
|
+
"""
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _fresh(seen: set, generate, rng: random.Random, tries: int = 100) -> str:
|
|
141
|
+
"""Draw from ``generate`` until unseen; fall back to a numbered suffix."""
|
|
142
|
+
for _ in range(tries):
|
|
143
|
+
candidate = generate(rng)
|
|
144
|
+
if candidate not in seen:
|
|
145
|
+
return candidate
|
|
146
|
+
base = generate(rng)
|
|
147
|
+
i = 2
|
|
148
|
+
while f"{base} {i}" in seen:
|
|
149
|
+
i += 1
|
|
150
|
+
return f"{base} {i}"
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
class NameAnonymizer(ColumnAnonymizer):
|
|
154
|
+
"""Replace values with realistic names from the bundled name library.
|
|
155
|
+
|
|
156
|
+
``style`` is one of ``first``, ``last``, ``first_last`` (default),
|
|
157
|
+
``first_middle_last``, ``last_first``. Replacements are unique within the
|
|
158
|
+
column; if the requested style cannot supply enough distinct
|
|
159
|
+
combinations, middle names are added automatically.
|
|
160
|
+
"""
|
|
161
|
+
|
|
162
|
+
STYLES = ("first", "last", "first_last", "first_middle_last", "last_first")
|
|
163
|
+
|
|
164
|
+
def __init__(self, style: str = "first_last"):
|
|
165
|
+
if style not in self.STYLES:
|
|
166
|
+
raise ValueError(
|
|
167
|
+
f"Unknown name style {style!r}. Choose from {self.STYLES}."
|
|
168
|
+
)
|
|
169
|
+
self.style = style
|
|
170
|
+
|
|
171
|
+
@staticmethod
|
|
172
|
+
def _capacity(style: str) -> int:
|
|
173
|
+
f, m, last = (
|
|
174
|
+
len(_names.FIRST_NAMES),
|
|
175
|
+
len(_names.MIDDLE_NAMES),
|
|
176
|
+
len(_names.LAST_NAMES),
|
|
177
|
+
)
|
|
178
|
+
return {
|
|
179
|
+
"first": f,
|
|
180
|
+
"last": last,
|
|
181
|
+
"first_last": f * last,
|
|
182
|
+
"last_first": f * last,
|
|
183
|
+
"first_middle_last": f * m * last,
|
|
184
|
+
}[style]
|
|
185
|
+
|
|
186
|
+
def _generate(self, style: str, rng: random.Random) -> str:
|
|
187
|
+
first = rng.choice(_names.FIRST_NAMES)
|
|
188
|
+
middle = rng.choice(_names.MIDDLE_NAMES)
|
|
189
|
+
last = rng.choice(_names.LAST_NAMES)
|
|
190
|
+
return {
|
|
191
|
+
"first": first,
|
|
192
|
+
"last": last,
|
|
193
|
+
"first_last": f"{first} {last}",
|
|
194
|
+
"first_middle_last": f"{first} {middle} {last}",
|
|
195
|
+
"last_first": f"{last}, {first}",
|
|
196
|
+
}[style]
|
|
197
|
+
|
|
198
|
+
def build_replacements(self, uniques, rng):
|
|
199
|
+
n = len(uniques)
|
|
200
|
+
style = self.style
|
|
201
|
+
# keep the collision rate low: escalate once past half capacity
|
|
202
|
+
if n > self._capacity(style) // 2:
|
|
203
|
+
style = "first_middle_last"
|
|
204
|
+
log.debug(
|
|
205
|
+
"NameAnonymizer: %d values exceed half capacity of %r; "
|
|
206
|
+
"using first_middle_last", n, self.style,
|
|
207
|
+
)
|
|
208
|
+
seen: set[str] = set()
|
|
209
|
+
out: list[str] = []
|
|
210
|
+
for _ in range(n):
|
|
211
|
+
name = _fresh(seen, lambda r: self._generate(style, r), rng)
|
|
212
|
+
seen.add(name)
|
|
213
|
+
out.append(name)
|
|
214
|
+
return out
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class SequentialIdAnonymizer(ColumnAnonymizer):
|
|
218
|
+
"""Replace values with sequential IDs: ``start, start+interval, ...``.
|
|
219
|
+
|
|
220
|
+
IDs are assigned in order of first appearance. With a ``prefix`` or
|
|
221
|
+
nonzero ``width`` the output is a zero-padded string (e.g. ``CUST-00120``);
|
|
222
|
+
otherwise it stays numeric.
|
|
223
|
+
"""
|
|
224
|
+
|
|
225
|
+
def __init__(self, start: int = 1, interval: int = 1, prefix: str = "", width: int = 0):
|
|
226
|
+
if interval == 0:
|
|
227
|
+
raise ValueError("interval must be nonzero")
|
|
228
|
+
self.start = int(start)
|
|
229
|
+
self.interval = int(interval)
|
|
230
|
+
self.prefix = str(prefix)
|
|
231
|
+
self.width = int(width)
|
|
232
|
+
|
|
233
|
+
def build_replacements(self, uniques, rng):
|
|
234
|
+
n = len(uniques)
|
|
235
|
+
# vectorized: start, start+interval, … in order of first appearance
|
|
236
|
+
values = self.start + np.arange(n, dtype=np.int64) * self.interval
|
|
237
|
+
if self.prefix or self.width:
|
|
238
|
+
# string formatting is per-unique (∝ uniques, not rows)
|
|
239
|
+
return np.array(
|
|
240
|
+
[f"{self.prefix}{int(v):0{self.width}d}" for v in values],
|
|
241
|
+
dtype=object,
|
|
242
|
+
)
|
|
243
|
+
return values
|
|
244
|
+
|
|
245
|
+
def _restore_dtype(self, original, result):
|
|
246
|
+
if not self.prefix and not self.width:
|
|
247
|
+
if result.isna().any():
|
|
248
|
+
return result.astype("Int64")
|
|
249
|
+
return result.astype("int64")
|
|
250
|
+
return result
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
class NumericJitterAnonymizer(ColumnAnonymizer):
|
|
254
|
+
"""Replace numbers with a random value within ±``pct`` of the original.
|
|
255
|
+
|
|
256
|
+
Defaults to ±20 %. Integer columns stay integers (the in-bound draw is
|
|
257
|
+
rounded to the nearest int, which for small magnitudes can nudge the result
|
|
258
|
+
a fraction beyond ±``pct``); ``round_to`` rounds floats to that many decimal
|
|
259
|
+
places. Note: a value of exactly 0 has no magnitude to jitter and is
|
|
260
|
+
returned unchanged.
|
|
261
|
+
|
|
262
|
+
This is additive noise, not a bijection: distinct originals that sit closer
|
|
263
|
+
together than the jitter can move them (e.g. adjacent integers under a small
|
|
264
|
+
``pct``) may land on the same replacement. The consistent mapping and the
|
|
265
|
+
±``pct`` bound always hold; strict distinct-to-distinct output does not.
|
|
266
|
+
"""
|
|
267
|
+
|
|
268
|
+
def __init__(self, pct: float = 0.20, round_to: int | None = None):
|
|
269
|
+
if not 0 < pct < 1:
|
|
270
|
+
raise ValueError("pct must be between 0 and 1 (e.g. 0.2 for ±20%)")
|
|
271
|
+
self.pct = float(pct)
|
|
272
|
+
self.round_to = round_to
|
|
273
|
+
|
|
274
|
+
def transform(self, series, rng=None):
|
|
275
|
+
if not pd.api.types.is_numeric_dtype(series) or pd.api.types.is_bool_dtype(series):
|
|
276
|
+
raise TypeError(
|
|
277
|
+
f"numeric_jitter needs a numeric column; {series.name!r} is "
|
|
278
|
+
f"{series.dtype}"
|
|
279
|
+
)
|
|
280
|
+
return super().transform(series, rng)
|
|
281
|
+
|
|
282
|
+
def build_replacements(self, uniques, rng):
|
|
283
|
+
vals = np.asarray(uniques, dtype=float)
|
|
284
|
+
integer = bool(np.all(np.mod(vals, 1) == 0)) if len(vals) else False
|
|
285
|
+
factors = _np_rng(rng).uniform(1 - self.pct, 1 + self.pct, size=len(vals))
|
|
286
|
+
jittered = vals * factors
|
|
287
|
+
if integer:
|
|
288
|
+
return np.rint(jittered).astype(np.int64)
|
|
289
|
+
if self.round_to is not None:
|
|
290
|
+
return np.round(jittered, self.round_to)
|
|
291
|
+
return jittered
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class DatetimeJitterAnonymizer(ColumnAnonymizer):
|
|
295
|
+
"""Shift each date/time by a random offset within ±``max_delta``.
|
|
296
|
+
|
|
297
|
+
``max_delta`` is anything :class:`pandas.Timedelta` accepts (``"7D"``,
|
|
298
|
+
``"12h"``, a number of ``unit`` s); it defaults to ±7 days. The offset is
|
|
299
|
+
drawn uniformly in whole ``unit`` steps (``"s"`` by default), so pick a
|
|
300
|
+
``unit`` no coarser than the resolution you want to keep — e.g.
|
|
301
|
+
``unit="D"`` jitters only by whole days. As with every anonymizer the
|
|
302
|
+
mapping is consistent: equal timestamps receive equal shifts, so the
|
|
303
|
+
column's shape survives. ``NaT`` is left untouched.
|
|
304
|
+
|
|
305
|
+
Non-datetime columns are coerced with :func:`pandas.to_datetime` first (so
|
|
306
|
+
date strings loaded from CSV work); a column that cannot be parsed as
|
|
307
|
+
dates raises :class:`TypeError`. Timezone-aware inputs keep their zone.
|
|
308
|
+
|
|
309
|
+
Like :class:`NumericJitterAnonymizer` this is bounded noise, not a
|
|
310
|
+
bijection: distinct timestamps closer together than the jitter window (and
|
|
311
|
+
at the chosen ``unit`` resolution) can shift onto the same replacement. Use
|
|
312
|
+
a wider ``max_delta`` or finer ``unit`` if that matters.
|
|
313
|
+
"""
|
|
314
|
+
|
|
315
|
+
def __init__(self, max_delta: str | int | float = "7D", unit: str = "s"):
|
|
316
|
+
self.max_delta = pd.Timedelta(max_delta)
|
|
317
|
+
if self.max_delta <= pd.Timedelta(0):
|
|
318
|
+
raise ValueError("max_delta must be positive (e.g. '7D' or '12h')")
|
|
319
|
+
self.unit = unit
|
|
320
|
+
# number of whole `unit` steps that fit in max_delta (the jitter span)
|
|
321
|
+
self._span = int(self.max_delta / pd.Timedelta(1, unit))
|
|
322
|
+
if self._span < 1:
|
|
323
|
+
raise ValueError(
|
|
324
|
+
f"max_delta {max_delta!r} is smaller than one {unit!r} step; "
|
|
325
|
+
"use a finer unit"
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
def transform(self, series, rng=None):
|
|
329
|
+
if not pd.api.types.is_datetime64_any_dtype(series):
|
|
330
|
+
try:
|
|
331
|
+
series = pd.to_datetime(series, errors="raise")
|
|
332
|
+
except (ValueError, TypeError) as exc:
|
|
333
|
+
raise TypeError(
|
|
334
|
+
f"datetime_jitter needs a datetime column; {series.name!r} "
|
|
335
|
+
f"is {series.dtype} and could not be parsed as dates ({exc})"
|
|
336
|
+
) from exc
|
|
337
|
+
return super().transform(series, rng)
|
|
338
|
+
|
|
339
|
+
def build_replacements(self, uniques, rng):
|
|
340
|
+
# vectorized: one uniform integer offset per unique, in whole units
|
|
341
|
+
offsets = _np_rng(rng).integers(
|
|
342
|
+
-self._span, self._span + 1, size=len(uniques)
|
|
343
|
+
)
|
|
344
|
+
base = pd.DatetimeIndex(uniques)
|
|
345
|
+
return base + pd.to_timedelta(offsets, unit=self.unit)
|
|
346
|
+
|
|
347
|
+
def _restore_dtype(self, original, result):
|
|
348
|
+
# the gather yields object/datetime64 values; normalize back to
|
|
349
|
+
# datetime64 (preserving tz-awareness of the coerced input)
|
|
350
|
+
return pd.to_datetime(result)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
class RandomStringAnonymizer(ColumnAnonymizer):
|
|
354
|
+
"""Replace values with random strings (unique within the column).
|
|
355
|
+
|
|
356
|
+
``charset`` is one of ``alphanumeric`` (default), ``letters``,
|
|
357
|
+
``digits``, ``hex``. ``prefix`` is prepended to every generated string.
|
|
358
|
+
"""
|
|
359
|
+
|
|
360
|
+
CHARSETS = {
|
|
361
|
+
"alphanumeric": string.ascii_lowercase + string.digits,
|
|
362
|
+
"letters": string.ascii_lowercase,
|
|
363
|
+
"digits": string.digits,
|
|
364
|
+
"hex": "0123456789abcdef",
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
def __init__(self, length: int = 8, charset: str = "alphanumeric", prefix: str = ""):
|
|
368
|
+
if charset not in self.CHARSETS:
|
|
369
|
+
raise ValueError(
|
|
370
|
+
f"Unknown charset {charset!r}. Choose from {tuple(self.CHARSETS)}."
|
|
371
|
+
)
|
|
372
|
+
if length < 1:
|
|
373
|
+
raise ValueError("length must be >= 1")
|
|
374
|
+
self.length = int(length)
|
|
375
|
+
self.charset = charset
|
|
376
|
+
self.prefix = str(prefix)
|
|
377
|
+
|
|
378
|
+
def _generate(self, rng: random.Random) -> str:
|
|
379
|
+
chars = self.CHARSETS[self.charset]
|
|
380
|
+
return self.prefix + "".join(rng.choice(chars) for _ in range(self.length))
|
|
381
|
+
|
|
382
|
+
def build_replacements(self, uniques, rng):
|
|
383
|
+
seen: set[str] = set()
|
|
384
|
+
out: list[str] = []
|
|
385
|
+
for _ in range(len(uniques)):
|
|
386
|
+
s = _fresh(seen, self._generate, rng)
|
|
387
|
+
seen.add(s)
|
|
388
|
+
out.append(s)
|
|
389
|
+
return out
|
|
390
|
+
|
|
391
|
+
|
|
392
|
+
KINDS: dict[str, type[ColumnAnonymizer]] = {
|
|
393
|
+
"names": NameAnonymizer,
|
|
394
|
+
"name": NameAnonymizer,
|
|
395
|
+
"sequential_id": SequentialIdAnonymizer,
|
|
396
|
+
"sequential": SequentialIdAnonymizer,
|
|
397
|
+
"seq": SequentialIdAnonymizer,
|
|
398
|
+
"numeric_jitter": NumericJitterAnonymizer,
|
|
399
|
+
"numbers": NumericJitterAnonymizer,
|
|
400
|
+
"jitter": NumericJitterAnonymizer,
|
|
401
|
+
"datetime_jitter": DatetimeJitterAnonymizer,
|
|
402
|
+
"datetime": DatetimeJitterAnonymizer,
|
|
403
|
+
"dates": DatetimeJitterAnonymizer,
|
|
404
|
+
"random_string": RandomStringAnonymizer,
|
|
405
|
+
"string": RandomStringAnonymizer,
|
|
406
|
+
"hex": RandomStringAnonymizer,
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def make_anonymizer(kind: str, **options) -> ColumnAnonymizer:
|
|
411
|
+
"""Build an anonymizer by kind name, e.g. ``make_anonymizer("seq", start=100)``.
|
|
412
|
+
|
|
413
|
+
The ``hex`` kind is shorthand for ``random_string`` with
|
|
414
|
+
``charset="hex"``.
|
|
415
|
+
"""
|
|
416
|
+
key = kind.strip().lower()
|
|
417
|
+
if key not in KINDS:
|
|
418
|
+
raise ValueError(f"Unknown anonymizer kind {kind!r}. Choose from {sorted(set(KINDS))}.")
|
|
419
|
+
if key == "hex":
|
|
420
|
+
options.setdefault("charset", "hex")
|
|
421
|
+
return KINDS[key](**options)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _coerce(value) -> ColumnAnonymizer:
|
|
425
|
+
if isinstance(value, ColumnAnonymizer):
|
|
426
|
+
return value
|
|
427
|
+
if isinstance(value, str):
|
|
428
|
+
return make_anonymizer(value)
|
|
429
|
+
if isinstance(value, tuple) and len(value) == 2 and isinstance(value[1], dict):
|
|
430
|
+
return make_anonymizer(value[0], **value[1])
|
|
431
|
+
if isinstance(value, dict):
|
|
432
|
+
opts = dict(value)
|
|
433
|
+
kind = opts.pop("kind")
|
|
434
|
+
return make_anonymizer(kind, **opts)
|
|
435
|
+
raise TypeError(
|
|
436
|
+
"Anonymizer spec must be a ColumnAnonymizer, a kind string, a "
|
|
437
|
+
"(kind, options) tuple, or a dict with a 'kind' key; "
|
|
438
|
+
f"got {value!r}"
|
|
439
|
+
)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def anonymize(
|
|
443
|
+
df: pd.DataFrame,
|
|
444
|
+
spec: Mapping[str, "ColumnAnonymizer | str | tuple | dict"],
|
|
445
|
+
seed: int | None = None,
|
|
446
|
+
) -> pd.DataFrame:
|
|
447
|
+
"""Return a copy of ``df`` with the columns named in ``spec`` anonymized.
|
|
448
|
+
|
|
449
|
+
``spec`` maps column name → anonymizer, where the anonymizer may be a
|
|
450
|
+
:class:`ColumnAnonymizer` instance, a kind string (``"names"``), a
|
|
451
|
+
``(kind, options)`` tuple (``("sequential_id", {"start": 1000})``), or a
|
|
452
|
+
dict (``{"kind": "hex", "length": 12}``). ``seed`` makes the run
|
|
453
|
+
reproducible. Columns not named in ``spec`` are returned unchanged.
|
|
454
|
+
"""
|
|
455
|
+
rng = random.Random(seed)
|
|
456
|
+
out = df.copy()
|
|
457
|
+
for col, raw in spec.items():
|
|
458
|
+
if col not in df.columns:
|
|
459
|
+
raise KeyError(f"Column {col!r} not found in DataFrame")
|
|
460
|
+
anonymizer = _coerce(raw)
|
|
461
|
+
log.info("anonymizing column %r with %s", col, type(anonymizer).__name__)
|
|
462
|
+
out[col] = anonymizer.transform(out[col], rng)
|
|
463
|
+
return out
|