sieve-crowd 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sieve_crowd-0.2.0/LICENSE +21 -0
- sieve_crowd-0.2.0/PKG-INFO +188 -0
- sieve_crowd-0.2.0/README.md +177 -0
- sieve_crowd-0.2.0/pyproject.toml +19 -0
- sieve_crowd-0.2.0/setup.cfg +4 -0
- sieve_crowd-0.2.0/sieve_crowd/__init__.py +16 -0
- sieve_crowd-0.2.0/sieve_crowd/__main__.py +3 -0
- sieve_crowd-0.2.0/sieve_crowd/cli.py +206 -0
- sieve_crowd-0.2.0/sieve_crowd/config.py +118 -0
- sieve_crowd-0.2.0/sieve_crowd/csvio.py +34 -0
- sieve_crowd-0.2.0/sieve_crowd/engine.py +205 -0
- sieve_crowd-0.2.0/sieve_crowd/generalize.py +209 -0
- sieve_crowd-0.2.0/sieve_crowd/hints.py +38 -0
- sieve_crowd-0.2.0/sieve_crowd/jsonio.py +106 -0
- sieve_crowd-0.2.0/sieve_crowd/measure.py +37 -0
- sieve_crowd-0.2.0/sieve_crowd/model.py +19 -0
- sieve_crowd-0.2.0/sieve_crowd/prng.py +31 -0
- sieve_crowd-0.2.0/sieve_crowd/pseudonym.py +19 -0
- sieve_crowd-0.2.0/sieve_crowd/report.py +139 -0
- sieve_crowd-0.2.0/sieve_crowd/search.py +145 -0
- sieve_crowd-0.2.0/sieve_crowd/tableio.py +51 -0
- sieve_crowd-0.2.0/sieve_crowd/xmlio.py +111 -0
- sieve_crowd-0.2.0/sieve_crowd.egg-info/PKG-INFO +188 -0
- sieve_crowd-0.2.0/sieve_crowd.egg-info/SOURCES.txt +32 -0
- sieve_crowd-0.2.0/sieve_crowd.egg-info/dependency_links.txt +1 -0
- sieve_crowd-0.2.0/sieve_crowd.egg-info/entry_points.txt +2 -0
- sieve_crowd-0.2.0/sieve_crowd.egg-info/top_level.txt +1 -0
- sieve_crowd-0.2.0/tests/test_cli.py +158 -0
- sieve_crowd-0.2.0/tests/test_engine.py +202 -0
- sieve_crowd-0.2.0/tests/test_engine_oracle.py +96 -0
- sieve_crowd-0.2.0/tests/test_generalize.py +101 -0
- sieve_crowd-0.2.0/tests/test_io_formats.py +361 -0
- sieve_crowd-0.2.0/tests/test_measure_and_prng.py +39 -0
- sieve_crowd-0.2.0/tests/test_vectors_and_version.py +41 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Kai IWASAKI
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sieve-crowd
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Make every record one of a crowd, and measure it: deterministic, offline k-anonymity for CSV, JSON and XML tables.
|
|
5
|
+
Author: Kai IWASAKI
|
|
6
|
+
License: MIT
|
|
7
|
+
Requires-Python: >=3.10
|
|
8
|
+
Description-Content-Type: text/markdown
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Dynamic: license-file
|
|
11
|
+
|
|
12
|
+
# Sieve Crowd v0.2.0
|
|
13
|
+
|
|
14
|
+
English | [ζ₯ζ¬θͺ](README.ja.md)
|
|
15
|
+
|
|
16
|
+
**Table data anonymization and privacy protection engine**
|
|
17
|
+
|
|
18
|
+
[](https://pypi.org/project/sieve-crowd/)
|
|
19
|
+
[](https://pypi.org/project/sieve-crowd/)
|
|
20
|
+
[](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml)
|
|
21
|
+
[](https://opensource.org/licenses/MIT)
|
|
22
|
+
|
|
23
|
+
> Sieve Crowd is a table-data processing engine that suppresses the risk of identifying individuals in tabular data, so that the data can be shared safely. Building on the principles of k-anonymity and l-diversity, it deterministically performs attribute rounding (generalization) and partial record removal (suppression).
|
|
24
|
+
|
|
25
|
+
A fully local, zero-dependency, deterministic (100% reproducible) mathematical transformation engine that anonymizes tabular data and disables re-identification. It extends the design philosophy of [Sieve-Core](https://github.com/neguseatama/sieve-core) into the tabular-data domain.
|
|
26
|
+
|
|
27
|
+
---
|
|
28
|
+
|
|
29
|
+
## π‘ Concept and design philosophy
|
|
30
|
+
|
|
31
|
+
Even when personal data is kept safely on your own machine, you sometimes have to provide or publish tabular data that contains it β or you face the risk of leakage. From a statistical standpoint, the combination of attributes such as age, postal code and sex always carries a risk of identifying individuals.
|
|
32
|
+
|
|
33
|
+
Consider, for example, a dataset of about 600 people. The risk structure becomes clear:
|
|
34
|
+
|
|
35
|
+
- **Before**: 574 of the 600 people (96%) were "alone" on the combination of age, postal code and sex β no one else shared their exact values, so an outsider could single them out easily.
|
|
36
|
+
- **How the transformation works**: rather than leaving these isolated records as they are, the engine groups similar records together (by rounding and generalization) so that everyone is hidden inside a safe crowd.
|
|
37
|
+
- **After**: every person belongs to a group of at least 5 (4 look-alikes besides themselves). Even an outsider who tries to identify someone can be right at most 1 in 5 times (20%).
|
|
38
|
+
|
|
39
|
+
Sieve Crowd does not rely on probabilistic inference or black-box AI. It transforms tabular data into a state in which no one stands out, deterministically, on the basis of the rigorous mathematical models of k-anonymity and l-diversity.
|
|
40
|
+
|
|
41
|
+
- **Deterministic approach**: the same input and settings always produce the same output. The result never depends on the execution environment, the clock, or random numbers.
|
|
42
|
+
- **A measurement, not a promise**: after the transformation, the finished table is re-counted to check whether it meets the target safety level. If it falls short, the reason is given on a receipt and the table is not written.
|
|
43
|
+
- **Explainability**: a receipt records the data processing performed for anonymization, so that what was done can be fully traced.
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## π§© Sieve series
|
|
48
|
+
|
|
49
|
+
Sieve is a family of engines that share three design principles β deterministic, zero-dependency, fully local. Each engine handles one observation or one removal, and they connect to each other:
|
|
50
|
+
|
|
51
|
+
| Package | Role | Connection |
|
|
52
|
+
| :--- | :--- | :--- |
|
|
53
|
+
| [sieve-core-engine](https://github.com/neguseatama/sieve-core) | Ecosystem base logic / plugin host | β |
|
|
54
|
+
| [sieve-lens](https://github.com/neguseatama/sieve-lens) | Observes invisible content and hidden prompts in documents | β |
|
|
55
|
+
| [sieve-scope](https://github.com/neguseatama/sieve-scope) | Detects structurally similar code pairs | β |
|
|
56
|
+
| [sieve-referee](https://github.com/neguseatama/sieve-referee) | Detects paraphrased or reused text pairs | β |
|
|
57
|
+
| [sieve-redact](https://github.com/neguseatama/sieve-redact) | Masks sensitive regions with a receipt (text & PNG) | β lens, β tray |
|
|
58
|
+
| [sieve-crowd](https://github.com/neguseatama/sieve-crowd) | Observes and anonymizes re-identification risk in tabular data (k-anonymity) | β |
|
|
59
|
+
| [sieve-tray](https://github.com/neguseatama/sieve-tray) | Desktop front-end that routes files to the right engine | β all engines |
|
|
60
|
+
|
|
61
|
+
---
|
|
62
|
+
|
|
63
|
+
## π₯ Main features
|
|
64
|
+
|
|
65
|
+
- **k-anonymity and l-diversity**
|
|
66
|
+
Attributes are generalized so that each record has at least kβ1 look-alikes, and the diversity of sensitive values is also guaranteed.
|
|
67
|
+
- **Zero dependencies, fully local**
|
|
68
|
+
Implemented with the Python standard library only. Data never leaves your computer (no account, offline, no upload).
|
|
69
|
+
- **Minimal generalization and suppression**
|
|
70
|
+
Data is not rounded more than necessary; resolution is preserved as much as possible while grouping. Records that still cannot form a group are safely removed (suppressed) rather than distorted.
|
|
71
|
+
- **Receipt for the record**
|
|
72
|
+
A machine-readable receipt records what was done (rules applied, suppression count, measurements). The receipt contains no personal information.
|
|
73
|
+
- **Supported formats**
|
|
74
|
+
CSV, JSON and XML (UTF-8). Excel is excluded to keep the tool dependency-free.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## π― Use cases
|
|
79
|
+
|
|
80
|
+
- Keeping data on hand for internal use or in-house sharing, stored and managed safely
|
|
81
|
+
- Providing survey or customer data to a university, agency, or analytics contractor
|
|
82
|
+
- Publishing statistics or open data from hospitals, schools, or local governments
|
|
83
|
+
- Feeding realistic test data into a development environment, safely
|
|
84
|
+
- Keeping a record of anonymization and processing for legal or audit purposes
|
|
85
|
+
|
|
86
|
+
---
|
|
87
|
+
|
|
88
|
+
## π¦ Install
|
|
89
|
+
|
|
90
|
+
```bash
|
|
91
|
+
pip install git+https://github.com/neguseatama/sieve-crowd.git
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
Or from a local clone:
|
|
95
|
+
|
|
96
|
+
```bash
|
|
97
|
+
git clone https://github.com/neguseatama/sieve-crowd.git
|
|
98
|
+
cd sieve-crowd
|
|
99
|
+
pip install .
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
---
|
|
103
|
+
|
|
104
|
+
## π» Quick start
|
|
105
|
+
|
|
106
|
+
### 1. Inspect the structure
|
|
107
|
+
|
|
108
|
+
Reads the file and writes a configuration file that proposes a role for each column (name, quasi-identifier, sensitive value, etc.).
|
|
109
|
+
|
|
110
|
+
```bash
|
|
111
|
+
sieve-crowd inspect people.csv --config-out config.json
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
### 2. Review and edit the configuration
|
|
115
|
+
|
|
116
|
+
Edit the generated `config.json` to define the role of each column.
|
|
117
|
+
|
|
118
|
+
### 3. Run anonymization
|
|
119
|
+
|
|
120
|
+
```bash
|
|
121
|
+
sieve-crowd run people.csv -c config.json -o people_safe.csv --receipt receipt.json
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
Input and output example:
|
|
125
|
+
|
|
126
|
+
```
|
|
127
|
+
people.csv people_safe.csv (rows shuffled)
|
|
128
|
+
name,birth_date,age,zip,sex,diagnosis age,zip,sex,diagnosis
|
|
129
|
+
Takahashi Hina,1986-11-02,40,1500052,M,none 20-24,15100**,F,cold
|
|
130
|
+
Kobayashi Haruto,2003-01-03,23,1000042,F,flu 20-24,16000**,M,none
|
|
131
|
+
Yamamoto Aoi,1971-10-04,55,1510061,M,flu 55-59,15100**,M,asthma
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
---
|
|
135
|
+
|
|
136
|
+
## βοΈ Settings and options
|
|
137
|
+
|
|
138
|
+
| Setting | Default | Meaning |
|
|
139
|
+
| :--- | :--- | :--- |
|
|
140
|
+
| `k` | 5 | Each record must have at least kβ1 look-alikes. Larger is safer but increases generalization. |
|
|
141
|
+
| `l` | optional | Every crowd must hold at least l distinct sensitive values, preventing "everyone has the same diagnosis". |
|
|
142
|
+
| `max_suppression` | 0.05 (5%) | The maximum fraction of records that may be removed (suppressed) instead of lowering the resolution of the whole table. |
|
|
143
|
+
| Role (`role`) | β | `id` (drop or pseudonymize) Β· `qi` (generalize) Β· `sensitive` (keep) Β· `keep` (keep) Β· `drop` (remove) |
|
|
144
|
+
|
|
145
|
+
Main command-line options:
|
|
146
|
+
|
|
147
|
+
- `--dry-run`: do not write data; only measure anonymization and assess risk
|
|
148
|
+
- `--lang ja`: issue a Japanese receipt (default is English; JSON output is also available)
|
|
149
|
+
- `--in-format` / `--out-format`: explicitly set the input/output format (csv, json, xml)
|
|
150
|
+
- `sieve-crowd keygen -o key.txt`: generate a cryptographic key for pseudonymization
|
|
151
|
+
|
|
152
|
+
Exit codes:
|
|
153
|
+
|
|
154
|
+
- `0`: success
|
|
155
|
+
- `1`: I/O error
|
|
156
|
+
- `2`: configuration error
|
|
157
|
+
- `3`: refused (the target anonymization level could not be reached; no output written)
|
|
158
|
+
|
|
159
|
+
---
|
|
160
|
+
|
|
161
|
+
## π¬ Tests
|
|
162
|
+
|
|
163
|
+
```bash
|
|
164
|
+
python -m unittest discover -s tests -v
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
All test suites run with no external dependencies (standard library only).
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## β οΈ Known limitations and caveats
|
|
172
|
+
|
|
173
|
+
1. **Model assumption** β Sieve Crowd protects against an attacker who knows the attributes you declared as `qi` (quasi-identifiers). It is not a universal shield against re-identification through auxiliary information you did not anticipate.
|
|
174
|
+
2. **Not a substitute for legal or privacy review** β It does not guarantee legal or regulatory compliance under privacy laws or guidelines; it is a tool that supports such review technically.
|
|
175
|
+
3. **Excel is not supported** β To keep the dependency-free principle, the binary `.xlsx` format is out of scope (use CSV / JSON / XML).
|
|
176
|
+
4. **Row shuffling** β To minimize re-identification risk, the output row order is deterministically shuffled.
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## π License
|
|
181
|
+
|
|
182
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
183
|
+
|
|
184
|
+
---
|
|
185
|
+
|
|
186
|
+
## π€ Author
|
|
187
|
+
|
|
188
|
+
* **Kai IWASAKI**
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
# Sieve Crowd v0.2.0
|
|
2
|
+
|
|
3
|
+
English | [ζ₯ζ¬θͺ](README.ja.md)
|
|
4
|
+
|
|
5
|
+
**Table data anonymization and privacy protection engine**
|
|
6
|
+
|
|
7
|
+
[](https://pypi.org/project/sieve-crowd/)
|
|
8
|
+
[](https://pypi.org/project/sieve-crowd/)
|
|
9
|
+
[](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml)
|
|
10
|
+
[](https://opensource.org/licenses/MIT)
|
|
11
|
+
|
|
12
|
+
> Sieve Crowd is a table-data processing engine that suppresses the risk of identifying individuals in tabular data, so that the data can be shared safely. Building on the principles of k-anonymity and l-diversity, it deterministically performs attribute rounding (generalization) and partial record removal (suppression).
|
|
13
|
+
|
|
14
|
+
A fully local, zero-dependency, deterministic (100% reproducible) mathematical transformation engine that anonymizes tabular data and disables re-identification. It extends the design philosophy of [Sieve-Core](https://github.com/neguseatama/sieve-core) into the tabular-data domain.
|
|
15
|
+
|
|
16
|
+
---
|
|
17
|
+
|
|
18
|
+
## π‘ Concept and design philosophy
|
|
19
|
+
|
|
20
|
+
Even when personal data is kept safely on your own machine, you sometimes have to provide or publish tabular data that contains it β or you face the risk of leakage. From a statistical standpoint, the combination of attributes such as age, postal code and sex always carries a risk of identifying individuals.
|
|
21
|
+
|
|
22
|
+
Consider, for example, a dataset of about 600 people. The risk structure becomes clear:
|
|
23
|
+
|
|
24
|
+
- **Before**: 574 of the 600 people (96%) were "alone" on the combination of age, postal code and sex β no one else shared their exact values, so an outsider could single them out easily.
|
|
25
|
+
- **How the transformation works**: rather than leaving these isolated records as they are, the engine groups similar records together (by rounding and generalization) so that everyone is hidden inside a safe crowd.
|
|
26
|
+
- **After**: every person belongs to a group of at least 5 (4 look-alikes besides themselves). Even an outsider who tries to identify someone can be right at most 1 in 5 times (20%).
|
|
27
|
+
|
|
28
|
+
Sieve Crowd does not rely on probabilistic inference or black-box AI. It transforms tabular data into a state in which no one stands out, deterministically, on the basis of the rigorous mathematical models of k-anonymity and l-diversity.
|
|
29
|
+
|
|
30
|
+
- **Deterministic approach**: the same input and settings always produce the same output. The result never depends on the execution environment, the clock, or random numbers.
|
|
31
|
+
- **A measurement, not a promise**: after the transformation, the finished table is re-counted to check whether it meets the target safety level. If it falls short, the reason is given on a receipt and the table is not written.
|
|
32
|
+
- **Explainability**: a receipt records the data processing performed for anonymization, so that what was done can be fully traced.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## π§© Sieve series
|
|
37
|
+
|
|
38
|
+
Sieve is a family of engines that share three design principles β deterministic, zero-dependency, fully local. Each engine handles one observation or one removal, and they connect to each other:
|
|
39
|
+
|
|
40
|
+
| Package | Role | Connection |
|
|
41
|
+
| :--- | :--- | :--- |
|
|
42
|
+
| [sieve-core-engine](https://github.com/neguseatama/sieve-core) | Ecosystem base logic / plugin host | β |
|
|
43
|
+
| [sieve-lens](https://github.com/neguseatama/sieve-lens) | Observes invisible content and hidden prompts in documents | β |
|
|
44
|
+
| [sieve-scope](https://github.com/neguseatama/sieve-scope) | Detects structurally similar code pairs | β |
|
|
45
|
+
| [sieve-referee](https://github.com/neguseatama/sieve-referee) | Detects paraphrased or reused text pairs | β |
|
|
46
|
+
| [sieve-redact](https://github.com/neguseatama/sieve-redact) | Masks sensitive regions with a receipt (text & PNG) | β lens, β tray |
|
|
47
|
+
| [sieve-crowd](https://github.com/neguseatama/sieve-crowd) | Observes and anonymizes re-identification risk in tabular data (k-anonymity) | β |
|
|
48
|
+
| [sieve-tray](https://github.com/neguseatama/sieve-tray) | Desktop front-end that routes files to the right engine | β all engines |
|
|
49
|
+
|
|
50
|
+
---
|
|
51
|
+
|
|
52
|
+
## π₯ Main features
|
|
53
|
+
|
|
54
|
+
- **k-anonymity and l-diversity**
|
|
55
|
+
Attributes are generalized so that each record has at least kβ1 look-alikes, and the diversity of sensitive values is also guaranteed.
|
|
56
|
+
- **Zero dependencies, fully local**
|
|
57
|
+
Implemented with the Python standard library only. Data never leaves your computer (no account, offline, no upload).
|
|
58
|
+
- **Minimal generalization and suppression**
|
|
59
|
+
Data is not rounded more than necessary; resolution is preserved as much as possible while grouping. Records that still cannot form a group are safely removed (suppressed) rather than distorted.
|
|
60
|
+
- **Receipt for the record**
|
|
61
|
+
A machine-readable receipt records what was done (rules applied, suppression count, measurements). The receipt contains no personal information.
|
|
62
|
+
- **Supported formats**
|
|
63
|
+
CSV, JSON and XML (UTF-8). Excel is excluded to keep the tool dependency-free.
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## π― Use cases
|
|
68
|
+
|
|
69
|
+
- Keeping data on hand for internal use or in-house sharing, stored and managed safely
|
|
70
|
+
- Providing survey or customer data to a university, agency, or analytics contractor
|
|
71
|
+
- Publishing statistics or open data from hospitals, schools, or local governments
|
|
72
|
+
- Feeding realistic test data into a development environment, safely
|
|
73
|
+
- Keeping a record of anonymization and processing for legal or audit purposes
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## π¦ Install
|
|
78
|
+
|
|
79
|
+
```bash
|
|
80
|
+
pip install git+https://github.com/neguseatama/sieve-crowd.git
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
Or from a local clone:
|
|
84
|
+
|
|
85
|
+
```bash
|
|
86
|
+
git clone https://github.com/neguseatama/sieve-crowd.git
|
|
87
|
+
cd sieve-crowd
|
|
88
|
+
pip install .
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## π» Quick start
|
|
94
|
+
|
|
95
|
+
### 1. Inspect the structure
|
|
96
|
+
|
|
97
|
+
Reads the file and writes a configuration file that proposes a role for each column (name, quasi-identifier, sensitive value, etc.).
|
|
98
|
+
|
|
99
|
+
```bash
|
|
100
|
+
sieve-crowd inspect people.csv --config-out config.json
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
### 2. Review and edit the configuration
|
|
104
|
+
|
|
105
|
+
Edit the generated `config.json` to define the role of each column.
|
|
106
|
+
|
|
107
|
+
### 3. Run anonymization
|
|
108
|
+
|
|
109
|
+
```bash
|
|
110
|
+
sieve-crowd run people.csv -c config.json -o people_safe.csv --receipt receipt.json
|
|
111
|
+
```
|
|
112
|
+
|
|
113
|
+
Input and output example:
|
|
114
|
+
|
|
115
|
+
```
|
|
116
|
+
people.csv people_safe.csv (rows shuffled)
|
|
117
|
+
name,birth_date,age,zip,sex,diagnosis age,zip,sex,diagnosis
|
|
118
|
+
Takahashi Hina,1986-11-02,40,1500052,M,none 20-24,15100**,F,cold
|
|
119
|
+
Kobayashi Haruto,2003-01-03,23,1000042,F,flu 20-24,16000**,M,none
|
|
120
|
+
Yamamoto Aoi,1971-10-04,55,1510061,M,flu 55-59,15100**,M,asthma
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
---
|
|
124
|
+
|
|
125
|
+
## βοΈ Settings and options
|
|
126
|
+
|
|
127
|
+
| Setting | Default | Meaning |
|
|
128
|
+
| :--- | :--- | :--- |
|
|
129
|
+
| `k` | 5 | Each record must have at least kβ1 look-alikes. Larger is safer but increases generalization. |
|
|
130
|
+
| `l` | optional | Every crowd must hold at least l distinct sensitive values, preventing "everyone has the same diagnosis". |
|
|
131
|
+
| `max_suppression` | 0.05 (5%) | The maximum fraction of records that may be removed (suppressed) instead of lowering the resolution of the whole table. |
|
|
132
|
+
| Role (`role`) | β | `id` (drop or pseudonymize) Β· `qi` (generalize) Β· `sensitive` (keep) Β· `keep` (keep) Β· `drop` (remove) |
|
|
133
|
+
|
|
134
|
+
Main command-line options:
|
|
135
|
+
|
|
136
|
+
- `--dry-run`: do not write data; only measure anonymization and assess risk
|
|
137
|
+
- `--lang ja`: issue a Japanese receipt (default is English; JSON output is also available)
|
|
138
|
+
- `--in-format` / `--out-format`: explicitly set the input/output format (csv, json, xml)
|
|
139
|
+
- `sieve-crowd keygen -o key.txt`: generate a cryptographic key for pseudonymization
|
|
140
|
+
|
|
141
|
+
Exit codes:
|
|
142
|
+
|
|
143
|
+
- `0`: success
|
|
144
|
+
- `1`: I/O error
|
|
145
|
+
- `2`: configuration error
|
|
146
|
+
- `3`: refused (the target anonymization level could not be reached; no output written)
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## π¬ Tests
|
|
151
|
+
|
|
152
|
+
```bash
|
|
153
|
+
python -m unittest discover -s tests -v
|
|
154
|
+
```
|
|
155
|
+
|
|
156
|
+
All test suites run with no external dependencies (standard library only).
|
|
157
|
+
|
|
158
|
+
---
|
|
159
|
+
|
|
160
|
+
## β οΈ Known limitations and caveats
|
|
161
|
+
|
|
162
|
+
1. **Model assumption** β Sieve Crowd protects against an attacker who knows the attributes you declared as `qi` (quasi-identifiers). It is not a universal shield against re-identification through auxiliary information you did not anticipate.
|
|
163
|
+
2. **Not a substitute for legal or privacy review** β It does not guarantee legal or regulatory compliance under privacy laws or guidelines; it is a tool that supports such review technically.
|
|
164
|
+
3. **Excel is not supported** β To keep the dependency-free principle, the binary `.xlsx` format is out of scope (use CSV / JSON / XML).
|
|
165
|
+
4. **Row shuffling** β To minimize re-identification risk, the output row order is deterministically shuffled.
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## π License
|
|
170
|
+
|
|
171
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
172
|
+
|
|
173
|
+
---
|
|
174
|
+
|
|
175
|
+
## π€ Author
|
|
176
|
+
|
|
177
|
+
* **Kai IWASAKI**
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "sieve-crowd"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Make every record one of a crowd, and measure it: deterministic, offline k-anonymity for CSV, JSON and XML tables."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "MIT" }
|
|
12
|
+
authors = [{ name = "Kai IWASAKI" }]
|
|
13
|
+
dependencies = []
|
|
14
|
+
|
|
15
|
+
[project.scripts]
|
|
16
|
+
sieve-crowd = "sieve_crowd.cli:main"
|
|
17
|
+
|
|
18
|
+
[tool.setuptools]
|
|
19
|
+
packages = ["sieve_crowd"]
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Sieve Crowd: make every record one of a crowd, and measure it.
|
|
2
|
+
|
|
3
|
+
Deterministic, offline, zero-dependency k-anonymity (with optional l-diversity)
|
|
4
|
+
for CSV, JSON and XML tables.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
__version__ = "0.2.0"
|
|
8
|
+
|
|
9
|
+
from .config import Config, parse_config, parse_key # noqa: E402
|
|
10
|
+
from .engine import Result, anonymize, inspect_table # noqa: E402
|
|
11
|
+
from .model import ConfigError, InputError, Role # noqa: E402
|
|
12
|
+
|
|
13
|
+
__all__ = [
|
|
14
|
+
"__version__", "anonymize", "inspect_table", "Result", "Config", "parse_config", "parse_key",
|
|
15
|
+
"ConfigError", "InputError", "Role",
|
|
16
|
+
]
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Command line: inspect / keygen / run. Cell values never appear in messages or logs."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import argparse
|
|
5
|
+
import json
|
|
6
|
+
import os
|
|
7
|
+
import secrets
|
|
8
|
+
import sys
|
|
9
|
+
import tempfile
|
|
10
|
+
from typing import Optional, Sequence
|
|
11
|
+
|
|
12
|
+
from . import __version__
|
|
13
|
+
from .config import parse_config, parse_key
|
|
14
|
+
from .engine import anonymize, inspect_table
|
|
15
|
+
from .hints import hint
|
|
16
|
+
from .model import ConfigError, InputError
|
|
17
|
+
from .tableio import FORMATS, detect_format, read_any, write_any
|
|
18
|
+
|
|
19
|
+
EXIT_OK, EXIT_IO, EXIT_CONFIG, EXIT_REFUSED = 0, 1, 2, 3
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _atomic_write(path: str, data: bytes, private: bool = False) -> None:
|
|
23
|
+
directory = os.path.dirname(os.path.abspath(path))
|
|
24
|
+
fd, tmp = tempfile.mkstemp(dir=directory, prefix=".sieve-crowd-")
|
|
25
|
+
try:
|
|
26
|
+
with os.fdopen(fd, "wb") as fh:
|
|
27
|
+
fh.write(data)
|
|
28
|
+
if private:
|
|
29
|
+
os.chmod(tmp, 0o600)
|
|
30
|
+
os.replace(tmp, path)
|
|
31
|
+
except BaseException:
|
|
32
|
+
try:
|
|
33
|
+
os.unlink(tmp)
|
|
34
|
+
except OSError:
|
|
35
|
+
pass
|
|
36
|
+
raise
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _parser() -> argparse.ArgumentParser:
|
|
40
|
+
p = argparse.ArgumentParser(prog="sieve-crowd",
|
|
41
|
+
description="Make every record one of a crowd of at least k, and measure it.")
|
|
42
|
+
p.add_argument("--version", action="version", version=f"sieve-crowd {__version__}")
|
|
43
|
+
sub = p.add_subparsers(dest="cmd", required=True)
|
|
44
|
+
|
|
45
|
+
i = sub.add_parser("inspect", help="describe the columns of a table (csv, json or xml) and write a config template")
|
|
46
|
+
i.add_argument("input")
|
|
47
|
+
i.add_argument("--delimiter", help="CSV only")
|
|
48
|
+
i.add_argument("--in-format", choices=FORMATS,
|
|
49
|
+
help="force the input format (default: from the file extension)")
|
|
50
|
+
i.add_argument("--lang", choices=["en", "ja"], default="en")
|
|
51
|
+
i.add_argument("--config-out", metavar="FILE", help="write a template with role '?' for every column")
|
|
52
|
+
i.add_argument("--force", action="store_true")
|
|
53
|
+
|
|
54
|
+
k = sub.add_parser("keygen", help="create a key for pseudonyms")
|
|
55
|
+
k.add_argument("-o", "--output", required=True)
|
|
56
|
+
k.add_argument("--force", action="store_true")
|
|
57
|
+
|
|
58
|
+
r = sub.add_parser("run", help="anonymize a table (csv, json or xml)")
|
|
59
|
+
r.add_argument("input")
|
|
60
|
+
r.add_argument("-c", "--config", required=True)
|
|
61
|
+
r.add_argument("-o", "--output")
|
|
62
|
+
r.add_argument("--dry-run", action="store_true", help="measure only; write no table")
|
|
63
|
+
r.add_argument("--delimiter", help="CSV only")
|
|
64
|
+
r.add_argument("--in-format", choices=FORMATS,
|
|
65
|
+
help="force the input format (default: from the file extension)")
|
|
66
|
+
r.add_argument("--out-format", choices=FORMATS,
|
|
67
|
+
help="force the output format (default: from the output extension)")
|
|
68
|
+
r.add_argument("--key-file")
|
|
69
|
+
r.add_argument("--k", type=int)
|
|
70
|
+
r.add_argument("--l", type=int)
|
|
71
|
+
r.add_argument("--max-suppression", type=float)
|
|
72
|
+
r.add_argument("--keep-order", action="store_true", help="relaxation: do not shuffle the rows (recorded)")
|
|
73
|
+
r.add_argument("--shuffle-seed", type=int, default=0)
|
|
74
|
+
r.add_argument("--lang", choices=["en", "ja"], default="en")
|
|
75
|
+
r.add_argument("--max-rows", type=int, default=200_000)
|
|
76
|
+
r.add_argument("--max-evals", type=int, default=20_000)
|
|
77
|
+
r.add_argument("--report-json", metavar="FILE")
|
|
78
|
+
r.add_argument("--quiet", action="store_true")
|
|
79
|
+
r.add_argument("--force", action="store_true")
|
|
80
|
+
return p
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _read(path: str) -> bytes:
|
|
84
|
+
if path == "-":
|
|
85
|
+
return sys.stdin.buffer.read()
|
|
86
|
+
with open(path, "rb") as fh:
|
|
87
|
+
return fh.read()
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _resolve_format(path: str, explicit: Optional[str], *, dash_default: str) -> str:
|
|
91
|
+
if explicit:
|
|
92
|
+
return explicit
|
|
93
|
+
if path == "-":
|
|
94
|
+
return dash_default
|
|
95
|
+
return detect_format(os.path.splitext(path)[1])
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def _delimiter(given: Optional[str], sides: set[str]) -> Optional[str]:
|
|
99
|
+
if given is None:
|
|
100
|
+
return None
|
|
101
|
+
if sides - {"csv"}:
|
|
102
|
+
raise InputError("--delimiter applies to csv only (the table is json/xml)")
|
|
103
|
+
return given
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _cmd_inspect(a: argparse.Namespace) -> int:
|
|
107
|
+
try:
|
|
108
|
+
in_fmt = _resolve_format(a.input, a.in_format, dash_default="csv")
|
|
109
|
+
delim = _delimiter(a.delimiter, {in_fmt})
|
|
110
|
+
header, rows = read_any(_read(a.input), in_fmt, delim)
|
|
111
|
+
except InputError as exc:
|
|
112
|
+
print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
|
|
113
|
+
except OSError as exc:
|
|
114
|
+
print(f"error: cannot read the input ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
115
|
+
info = inspect_table(header, rows)
|
|
116
|
+
en = a.lang == "en"
|
|
117
|
+
print(f"records: {len(rows)}" if en else f"γ¬γ³γΌγζ°: {len(rows)}")
|
|
118
|
+
hints = {}
|
|
119
|
+
for c in info:
|
|
120
|
+
role, text_en, text_ja = hint(c["name"], c, len(rows))
|
|
121
|
+
hints[c["name"]] = (role, text_en if en else text_ja)
|
|
122
|
+
shown = f" {c['name']}: {c['distinct']} distinct, {c['empty']} empty, {c['unique_ratio']:.0%} unique values, looks like {c['shape']}" \
|
|
123
|
+
if en else f" {c['name']}: η°γͺγε€ {c['distinct']}γη©Ί {c['empty']}γδΈζγͺε€ {c['unique_ratio']:.0%}γε½’γ― {c['shape']}"
|
|
124
|
+
print(shown)
|
|
125
|
+
if role:
|
|
126
|
+
print(f" -> {hints[c['name']][1]}")
|
|
127
|
+
if a.config_out:
|
|
128
|
+
if os.path.exists(a.config_out) and not a.force:
|
|
129
|
+
print("error: the config file exists (use --force)", file=sys.stderr); return EXIT_IO
|
|
130
|
+
template = {"k": 5, "l": 1, "max_suppression": 0.05,
|
|
131
|
+
"columns": {c["name"]: ({"role": "?", "_hint": hints[c["name"]][1]} if hints[c["name"]][0]
|
|
132
|
+
else {"role": "?"}) for c in info}}
|
|
133
|
+
try:
|
|
134
|
+
_atomic_write(a.config_out, (json.dumps(template, ensure_ascii=False, indent=2) + "\n").encode())
|
|
135
|
+
except OSError as exc:
|
|
136
|
+
print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
137
|
+
return EXIT_OK
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _cmd_keygen(a: argparse.Namespace) -> int:
|
|
141
|
+
if os.path.exists(a.output) and not a.force:
|
|
142
|
+
print("error: the key file exists (use --force)", file=sys.stderr); return EXIT_IO
|
|
143
|
+
try:
|
|
144
|
+
_atomic_write(a.output, (secrets.token_hex(32) + "\n").encode(), private=True)
|
|
145
|
+
except OSError as exc:
|
|
146
|
+
print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
147
|
+
return EXIT_OK
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _cmd_run(a: argparse.Namespace) -> int:
|
|
151
|
+
if not a.dry_run and not a.output:
|
|
152
|
+
print("error: give -o/--output or --dry-run", file=sys.stderr); return EXIT_CONFIG
|
|
153
|
+
if a.output and a.input != "-" and os.path.realpath(a.output) == os.path.realpath(a.input):
|
|
154
|
+
print("error: the output must not be the input file", file=sys.stderr); return EXIT_CONFIG
|
|
155
|
+
if a.output and os.path.exists(a.output) and not a.force:
|
|
156
|
+
print("error: output exists (use --force to replace it)", file=sys.stderr); return EXIT_IO
|
|
157
|
+
try:
|
|
158
|
+
in_fmt = _resolve_format(a.input, a.in_format, dash_default="csv")
|
|
159
|
+
out_fmt = _resolve_format(a.output, a.out_format, dash_default="csv") if a.output else None
|
|
160
|
+
delim = _delimiter(a.delimiter, {in_fmt} | (set() if out_fmt is None else {out_fmt}))
|
|
161
|
+
except InputError as exc:
|
|
162
|
+
print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
|
|
163
|
+
try:
|
|
164
|
+
with open(a.config, "rb") as fh:
|
|
165
|
+
raw = json.loads(fh.read().decode("utf-8-sig"))
|
|
166
|
+
key = parse_key(open(a.key_file, encoding="utf-8").read()) if a.key_file else None
|
|
167
|
+
cfg = parse_config(raw, k=a.k, l=a.l, max_suppression=a.max_suppression,
|
|
168
|
+
shuffle=not a.keep_order, shuffle_seed=a.shuffle_seed, lang=a.lang,
|
|
169
|
+
max_rows=a.max_rows, max_evals=a.max_evals, key=key)
|
|
170
|
+
except ConfigError as exc:
|
|
171
|
+
print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
|
|
172
|
+
except (json.JSONDecodeError, UnicodeDecodeError):
|
|
173
|
+
print("error: the configuration is not valid JSON", file=sys.stderr); return EXIT_CONFIG
|
|
174
|
+
except OSError as exc:
|
|
175
|
+
print(f"error: cannot read the configuration or key ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
176
|
+
try:
|
|
177
|
+
header, rows = read_any(_read(a.input), in_fmt, delim)
|
|
178
|
+
except InputError as exc:
|
|
179
|
+
print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
|
|
180
|
+
except OSError as exc:
|
|
181
|
+
print(f"error: cannot read the input ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
182
|
+
try:
|
|
183
|
+
result = anonymize(header, rows, cfg, io=(in_fmt, out_fmt))
|
|
184
|
+
except ConfigError as exc:
|
|
185
|
+
print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
|
|
186
|
+
if not a.quiet:
|
|
187
|
+
print(result.report.text(cfg.lang), file=sys.stderr)
|
|
188
|
+
try:
|
|
189
|
+
if a.report_json:
|
|
190
|
+
_atomic_write(a.report_json, (result.report.to_json() + "\n").encode("utf-8"))
|
|
191
|
+
if result.status == "refused":
|
|
192
|
+
return EXIT_REFUSED
|
|
193
|
+
if not a.dry_run:
|
|
194
|
+
data = write_any(result.header, result.rows, out_fmt, delim)
|
|
195
|
+
if a.output == "-":
|
|
196
|
+
sys.stdout.buffer.write(data); sys.stdout.buffer.flush()
|
|
197
|
+
else:
|
|
198
|
+
_atomic_write(a.output, data)
|
|
199
|
+
except OSError as exc:
|
|
200
|
+
print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
|
|
201
|
+
return EXIT_OK
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def main(argv: Optional[Sequence[str]] = None) -> int:
|
|
205
|
+
a = _parser().parse_args(argv)
|
|
206
|
+
return {"inspect": _cmd_inspect, "keygen": _cmd_keygen, "run": _cmd_run}[a.cmd](a)
|