sieve-crowd 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (34) hide show
  1. sieve_crowd-0.2.0/LICENSE +21 -0
  2. sieve_crowd-0.2.0/PKG-INFO +188 -0
  3. sieve_crowd-0.2.0/README.md +177 -0
  4. sieve_crowd-0.2.0/pyproject.toml +19 -0
  5. sieve_crowd-0.2.0/setup.cfg +4 -0
  6. sieve_crowd-0.2.0/sieve_crowd/__init__.py +16 -0
  7. sieve_crowd-0.2.0/sieve_crowd/__main__.py +3 -0
  8. sieve_crowd-0.2.0/sieve_crowd/cli.py +206 -0
  9. sieve_crowd-0.2.0/sieve_crowd/config.py +118 -0
  10. sieve_crowd-0.2.0/sieve_crowd/csvio.py +34 -0
  11. sieve_crowd-0.2.0/sieve_crowd/engine.py +205 -0
  12. sieve_crowd-0.2.0/sieve_crowd/generalize.py +209 -0
  13. sieve_crowd-0.2.0/sieve_crowd/hints.py +38 -0
  14. sieve_crowd-0.2.0/sieve_crowd/jsonio.py +106 -0
  15. sieve_crowd-0.2.0/sieve_crowd/measure.py +37 -0
  16. sieve_crowd-0.2.0/sieve_crowd/model.py +19 -0
  17. sieve_crowd-0.2.0/sieve_crowd/prng.py +31 -0
  18. sieve_crowd-0.2.0/sieve_crowd/pseudonym.py +19 -0
  19. sieve_crowd-0.2.0/sieve_crowd/report.py +139 -0
  20. sieve_crowd-0.2.0/sieve_crowd/search.py +145 -0
  21. sieve_crowd-0.2.0/sieve_crowd/tableio.py +51 -0
  22. sieve_crowd-0.2.0/sieve_crowd/xmlio.py +111 -0
  23. sieve_crowd-0.2.0/sieve_crowd.egg-info/PKG-INFO +188 -0
  24. sieve_crowd-0.2.0/sieve_crowd.egg-info/SOURCES.txt +32 -0
  25. sieve_crowd-0.2.0/sieve_crowd.egg-info/dependency_links.txt +1 -0
  26. sieve_crowd-0.2.0/sieve_crowd.egg-info/entry_points.txt +2 -0
  27. sieve_crowd-0.2.0/sieve_crowd.egg-info/top_level.txt +1 -0
  28. sieve_crowd-0.2.0/tests/test_cli.py +158 -0
  29. sieve_crowd-0.2.0/tests/test_engine.py +202 -0
  30. sieve_crowd-0.2.0/tests/test_engine_oracle.py +96 -0
  31. sieve_crowd-0.2.0/tests/test_generalize.py +101 -0
  32. sieve_crowd-0.2.0/tests/test_io_formats.py +361 -0
  33. sieve_crowd-0.2.0/tests/test_measure_and_prng.py +39 -0
  34. sieve_crowd-0.2.0/tests/test_vectors_and_version.py +41 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Kai IWASAKI
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,188 @@
1
+ Metadata-Version: 2.4
2
+ Name: sieve-crowd
3
+ Version: 0.2.0
4
+ Summary: Make every record one of a crowd, and measure it: deterministic, offline k-anonymity for CSV, JSON and XML tables.
5
+ Author: Kai IWASAKI
6
+ License: MIT
7
+ Requires-Python: >=3.10
8
+ Description-Content-Type: text/markdown
9
+ License-File: LICENSE
10
+ Dynamic: license-file
11
+
12
+ # Sieve Crowd v0.2.0
13
+
14
+ English | [ζ—₯本θͺž](README.ja.md)
15
+
16
+ **Table data anonymization and privacy protection engine**
17
+
18
+ [![PyPI Version](https://img.shields.io/pypi/v/sieve-crowd.svg)](https://pypi.org/project/sieve-crowd/)
19
+ [![Python Version](https://img.shields.io/pypi/pyversions/sieve-crowd.svg)](https://pypi.org/project/sieve-crowd/)
20
+ [![CI](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml/badge.svg)](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml)
21
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
22
+
23
+ > Sieve Crowd is a table-data processing engine that suppresses the risk of identifying individuals in tabular data, so that the data can be shared safely. Building on the principles of k-anonymity and l-diversity, it deterministically performs attribute rounding (generalization) and partial record removal (suppression).
24
+
25
+ A fully local, zero-dependency, deterministic (100% reproducible) mathematical transformation engine that anonymizes tabular data and disables re-identification. It extends the design philosophy of [Sieve-Core](https://github.com/neguseatama/sieve-core) into the tabular-data domain.
26
+
27
+ ---
28
+
29
+ ## πŸ’‘ Concept and design philosophy
30
+
31
+ Even when personal data is kept safely on your own machine, you sometimes have to provide or publish tabular data that contains it β€” or you face the risk of leakage. From a statistical standpoint, the combination of attributes such as age, postal code and sex always carries a risk of identifying individuals.
32
+
33
+ Consider, for example, a dataset of about 600 people. The risk structure becomes clear:
34
+
35
+ - **Before**: 574 of the 600 people (96%) were "alone" on the combination of age, postal code and sex β€” no one else shared their exact values, so an outsider could single them out easily.
36
+ - **How the transformation works**: rather than leaving these isolated records as they are, the engine groups similar records together (by rounding and generalization) so that everyone is hidden inside a safe crowd.
37
+ - **After**: every person belongs to a group of at least 5 (4 look-alikes besides themselves). Even an outsider who tries to identify someone can be right at most 1 in 5 times (20%).
38
+
39
+ Sieve Crowd does not rely on probabilistic inference or black-box AI. It transforms tabular data into a state in which no one stands out, deterministically, on the basis of the rigorous mathematical models of k-anonymity and l-diversity.
40
+
41
+ - **Deterministic approach**: the same input and settings always produce the same output. The result never depends on the execution environment, the clock, or random numbers.
42
+ - **A measurement, not a promise**: after the transformation, the finished table is re-counted to check whether it meets the target safety level. If it falls short, the reason is given on a receipt and the table is not written.
43
+ - **Explainability**: a receipt records the data processing performed for anonymization, so that what was done can be fully traced.
44
+
45
+ ---
46
+
47
+ ## 🧩 Sieve series
48
+
49
+ Sieve is a family of engines that share three design principles β€” deterministic, zero-dependency, fully local. Each engine handles one observation or one removal, and they connect to each other:
50
+
51
+ | Package | Role | Connection |
52
+ | :--- | :--- | :--- |
53
+ | [sieve-core-engine](https://github.com/neguseatama/sieve-core) | Ecosystem base logic / plugin host | β€” |
54
+ | [sieve-lens](https://github.com/neguseatama/sieve-lens) | Observes invisible content and hidden prompts in documents | β€” |
55
+ | [sieve-scope](https://github.com/neguseatama/sieve-scope) | Detects structurally similar code pairs | β€” |
56
+ | [sieve-referee](https://github.com/neguseatama/sieve-referee) | Detects paraphrased or reused text pairs | β€” |
57
+ | [sieve-redact](https://github.com/neguseatama/sieve-redact) | Masks sensitive regions with a receipt (text & PNG) | ← lens, β†’ tray |
58
+ | [sieve-crowd](https://github.com/neguseatama/sieve-crowd) | Observes and anonymizes re-identification risk in tabular data (k-anonymity) | β€” |
59
+ | [sieve-tray](https://github.com/neguseatama/sieve-tray) | Desktop front-end that routes files to the right engine | ← all engines |
60
+
61
+ ---
62
+
63
+ ## πŸ”₯ Main features
64
+
65
+ - **k-anonymity and l-diversity**
66
+ Attributes are generalized so that each record has at least kβˆ’1 look-alikes, and the diversity of sensitive values is also guaranteed.
67
+ - **Zero dependencies, fully local**
68
+ Implemented with the Python standard library only. Data never leaves your computer (no account, offline, no upload).
69
+ - **Minimal generalization and suppression**
70
+ Data is not rounded more than necessary; resolution is preserved as much as possible while grouping. Records that still cannot form a group are safely removed (suppressed) rather than distorted.
71
+ - **Receipt for the record**
72
+ A machine-readable receipt records what was done (rules applied, suppression count, measurements). The receipt contains no personal information.
73
+ - **Supported formats**
74
+ CSV, JSON and XML (UTF-8). Excel is excluded to keep the tool dependency-free.
75
+
76
+ ---
77
+
78
+ ## 🎯 Use cases
79
+
80
+ - Keeping data on hand for internal use or in-house sharing, stored and managed safely
81
+ - Providing survey or customer data to a university, agency, or analytics contractor
82
+ - Publishing statistics or open data from hospitals, schools, or local governments
83
+ - Feeding realistic test data into a development environment, safely
84
+ - Keeping a record of anonymization and processing for legal or audit purposes
85
+
86
+ ---
87
+
88
+ ## πŸ“¦ Install
89
+
90
+ ```bash
91
+ pip install git+https://github.com/neguseatama/sieve-crowd.git
92
+ ```
93
+
94
+ Or from a local clone:
95
+
96
+ ```bash
97
+ git clone https://github.com/neguseatama/sieve-crowd.git
98
+ cd sieve-crowd
99
+ pip install .
100
+ ```
101
+
102
+ ---
103
+
104
+ ## πŸ’» Quick start
105
+
106
+ ### 1. Inspect the structure
107
+
108
+ Reads the file and writes a configuration file that proposes a role for each column (name, quasi-identifier, sensitive value, etc.).
109
+
110
+ ```bash
111
+ sieve-crowd inspect people.csv --config-out config.json
112
+ ```
113
+
114
+ ### 2. Review and edit the configuration
115
+
116
+ Edit the generated `config.json` to define the role of each column.
117
+
118
+ ### 3. Run anonymization
119
+
120
+ ```bash
121
+ sieve-crowd run people.csv -c config.json -o people_safe.csv --receipt receipt.json
122
+ ```
123
+
124
+ Input and output example:
125
+
126
+ ```
127
+ people.csv people_safe.csv (rows shuffled)
128
+ name,birth_date,age,zip,sex,diagnosis age,zip,sex,diagnosis
129
+ Takahashi Hina,1986-11-02,40,1500052,M,none 20-24,15100**,F,cold
130
+ Kobayashi Haruto,2003-01-03,23,1000042,F,flu 20-24,16000**,M,none
131
+ Yamamoto Aoi,1971-10-04,55,1510061,M,flu 55-59,15100**,M,asthma
132
+ ```
133
+
134
+ ---
135
+
136
+ ## βš™οΈ Settings and options
137
+
138
+ | Setting | Default | Meaning |
139
+ | :--- | :--- | :--- |
140
+ | `k` | 5 | Each record must have at least kβˆ’1 look-alikes. Larger is safer but increases generalization. |
141
+ | `l` | optional | Every crowd must hold at least l distinct sensitive values, preventing "everyone has the same diagnosis". |
142
+ | `max_suppression` | 0.05 (5%) | The maximum fraction of records that may be removed (suppressed) instead of lowering the resolution of the whole table. |
143
+ | Role (`role`) | β€” | `id` (drop or pseudonymize) Β· `qi` (generalize) Β· `sensitive` (keep) Β· `keep` (keep) Β· `drop` (remove) |
144
+
145
+ Main command-line options:
146
+
147
+ - `--dry-run`: do not write data; only measure anonymization and assess risk
148
+ - `--lang ja`: issue a Japanese receipt (default is English; JSON output is also available)
149
+ - `--in-format` / `--out-format`: explicitly set the input/output format (csv, json, xml)
150
+ - `sieve-crowd keygen -o key.txt`: generate a cryptographic key for pseudonymization
151
+
152
+ Exit codes:
153
+
154
+ - `0`: success
155
+ - `1`: I/O error
156
+ - `2`: configuration error
157
+ - `3`: refused (the target anonymization level could not be reached; no output written)
158
+
159
+ ---
160
+
161
+ ## πŸ”¬ Tests
162
+
163
+ ```bash
164
+ python -m unittest discover -s tests -v
165
+ ```
166
+
167
+ All test suites run with no external dependencies (standard library only).
168
+
169
+ ---
170
+
171
+ ## ⚠️ Known limitations and caveats
172
+
173
+ 1. **Model assumption** β€” Sieve Crowd protects against an attacker who knows the attributes you declared as `qi` (quasi-identifiers). It is not a universal shield against re-identification through auxiliary information you did not anticipate.
174
+ 2. **Not a substitute for legal or privacy review** β€” It does not guarantee legal or regulatory compliance under privacy laws or guidelines; it is a tool that supports such review technically.
175
+ 3. **Excel is not supported** β€” To keep the dependency-free principle, the binary `.xlsx` format is out of scope (use CSV / JSON / XML).
176
+ 4. **Row shuffling** β€” To minimize re-identification risk, the output row order is deterministically shuffled.
177
+
178
+ ---
179
+
180
+ ## πŸ“„ License
181
+
182
+ MIT License. See [LICENSE](LICENSE) for details.
183
+
184
+ ---
185
+
186
+ ## πŸ‘€ Author
187
+
188
+ * **Kai IWASAKI**
@@ -0,0 +1,177 @@
1
+ # Sieve Crowd v0.2.0
2
+
3
+ English | [ζ—₯本θͺž](README.ja.md)
4
+
5
+ **Table data anonymization and privacy protection engine**
6
+
7
+ [![PyPI Version](https://img.shields.io/pypi/v/sieve-crowd.svg)](https://pypi.org/project/sieve-crowd/)
8
+ [![Python Version](https://img.shields.io/pypi/pyversions/sieve-crowd.svg)](https://pypi.org/project/sieve-crowd/)
9
+ [![CI](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml/badge.svg)](https://github.com/neguseatama/sieve-crowd/actions/workflows/test.yml)
10
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT)
11
+
12
+ > Sieve Crowd is a table-data processing engine that suppresses the risk of identifying individuals in tabular data, so that the data can be shared safely. Building on the principles of k-anonymity and l-diversity, it deterministically performs attribute rounding (generalization) and partial record removal (suppression).
13
+
14
+ A fully local, zero-dependency, deterministic (100% reproducible) mathematical transformation engine that anonymizes tabular data and disables re-identification. It extends the design philosophy of [Sieve-Core](https://github.com/neguseatama/sieve-core) into the tabular-data domain.
15
+
16
+ ---
17
+
18
+ ## πŸ’‘ Concept and design philosophy
19
+
20
+ Even when personal data is kept safely on your own machine, you sometimes have to provide or publish tabular data that contains it β€” or you face the risk of leakage. From a statistical standpoint, the combination of attributes such as age, postal code and sex always carries a risk of identifying individuals.
21
+
22
+ Consider, for example, a dataset of about 600 people. The risk structure becomes clear:
23
+
24
+ - **Before**: 574 of the 600 people (96%) were "alone" on the combination of age, postal code and sex β€” no one else shared their exact values, so an outsider could single them out easily.
25
+ - **How the transformation works**: rather than leaving these isolated records as they are, the engine groups similar records together (by rounding and generalization) so that everyone is hidden inside a safe crowd.
26
+ - **After**: every person belongs to a group of at least 5 (4 look-alikes besides themselves). Even an outsider who tries to identify someone can be right at most 1 in 5 times (20%).
27
+
28
+ Sieve Crowd does not rely on probabilistic inference or black-box AI. It transforms tabular data into a state in which no one stands out, deterministically, on the basis of the rigorous mathematical models of k-anonymity and l-diversity.
29
+
30
+ - **Deterministic approach**: the same input and settings always produce the same output. The result never depends on the execution environment, the clock, or random numbers.
31
+ - **A measurement, not a promise**: after the transformation, the finished table is re-counted to check whether it meets the target safety level. If it falls short, the reason is given on a receipt and the table is not written.
32
+ - **Explainability**: a receipt records the data processing performed for anonymization, so that what was done can be fully traced.
33
+
34
+ ---
35
+
36
+ ## 🧩 Sieve series
37
+
38
+ Sieve is a family of engines that share three design principles β€” deterministic, zero-dependency, fully local. Each engine handles one observation or one removal, and they connect to each other:
39
+
40
+ | Package | Role | Connection |
41
+ | :--- | :--- | :--- |
42
+ | [sieve-core-engine](https://github.com/neguseatama/sieve-core) | Ecosystem base logic / plugin host | β€” |
43
+ | [sieve-lens](https://github.com/neguseatama/sieve-lens) | Observes invisible content and hidden prompts in documents | β€” |
44
+ | [sieve-scope](https://github.com/neguseatama/sieve-scope) | Detects structurally similar code pairs | β€” |
45
+ | [sieve-referee](https://github.com/neguseatama/sieve-referee) | Detects paraphrased or reused text pairs | β€” |
46
+ | [sieve-redact](https://github.com/neguseatama/sieve-redact) | Masks sensitive regions with a receipt (text & PNG) | ← lens, β†’ tray |
47
+ | [sieve-crowd](https://github.com/neguseatama/sieve-crowd) | Observes and anonymizes re-identification risk in tabular data (k-anonymity) | β€” |
48
+ | [sieve-tray](https://github.com/neguseatama/sieve-tray) | Desktop front-end that routes files to the right engine | ← all engines |
49
+
50
+ ---
51
+
52
+ ## πŸ”₯ Main features
53
+
54
+ - **k-anonymity and l-diversity**
55
+ Attributes are generalized so that each record has at least kβˆ’1 look-alikes, and the diversity of sensitive values is also guaranteed.
56
+ - **Zero dependencies, fully local**
57
+ Implemented with the Python standard library only. Data never leaves your computer (no account, offline, no upload).
58
+ - **Minimal generalization and suppression**
59
+ Data is not rounded more than necessary; resolution is preserved as much as possible while grouping. Records that still cannot form a group are safely removed (suppressed) rather than distorted.
60
+ - **Receipt for the record**
61
+ A machine-readable receipt records what was done (rules applied, suppression count, measurements). The receipt contains no personal information.
62
+ - **Supported formats**
63
+ CSV, JSON and XML (UTF-8). Excel is excluded to keep the tool dependency-free.
64
+
65
+ ---
66
+
67
+ ## 🎯 Use cases
68
+
69
+ - Keeping data on hand for internal use or in-house sharing, stored and managed safely
70
+ - Providing survey or customer data to a university, agency, or analytics contractor
71
+ - Publishing statistics or open data from hospitals, schools, or local governments
72
+ - Feeding realistic test data into a development environment, safely
73
+ - Keeping a record of anonymization and processing for legal or audit purposes
74
+
75
+ ---
76
+
77
+ ## πŸ“¦ Install
78
+
79
+ ```bash
80
+ pip install git+https://github.com/neguseatama/sieve-crowd.git
81
+ ```
82
+
83
+ Or from a local clone:
84
+
85
+ ```bash
86
+ git clone https://github.com/neguseatama/sieve-crowd.git
87
+ cd sieve-crowd
88
+ pip install .
89
+ ```
90
+
91
+ ---
92
+
93
+ ## πŸ’» Quick start
94
+
95
+ ### 1. Inspect the structure
96
+
97
+ Reads the file and writes a configuration file that proposes a role for each column (name, quasi-identifier, sensitive value, etc.).
98
+
99
+ ```bash
100
+ sieve-crowd inspect people.csv --config-out config.json
101
+ ```
102
+
103
+ ### 2. Review and edit the configuration
104
+
105
+ Edit the generated `config.json` to define the role of each column.
106
+
107
+ ### 3. Run anonymization
108
+
109
+ ```bash
110
+ sieve-crowd run people.csv -c config.json -o people_safe.csv --receipt receipt.json
111
+ ```
112
+
113
+ Input and output example:
114
+
115
+ ```
116
+ people.csv people_safe.csv (rows shuffled)
117
+ name,birth_date,age,zip,sex,diagnosis age,zip,sex,diagnosis
118
+ Takahashi Hina,1986-11-02,40,1500052,M,none 20-24,15100**,F,cold
119
+ Kobayashi Haruto,2003-01-03,23,1000042,F,flu 20-24,16000**,M,none
120
+ Yamamoto Aoi,1971-10-04,55,1510061,M,flu 55-59,15100**,M,asthma
121
+ ```
122
+
123
+ ---
124
+
125
+ ## βš™οΈ Settings and options
126
+
127
+ | Setting | Default | Meaning |
128
+ | :--- | :--- | :--- |
129
+ | `k` | 5 | Each record must have at least kβˆ’1 look-alikes. Larger is safer but increases generalization. |
130
+ | `l` | optional | Every crowd must hold at least l distinct sensitive values, preventing "everyone has the same diagnosis". |
131
+ | `max_suppression` | 0.05 (5%) | The maximum fraction of records that may be removed (suppressed) instead of lowering the resolution of the whole table. |
132
+ | Role (`role`) | β€” | `id` (drop or pseudonymize) Β· `qi` (generalize) Β· `sensitive` (keep) Β· `keep` (keep) Β· `drop` (remove) |
133
+
134
+ Main command-line options:
135
+
136
+ - `--dry-run`: do not write data; only measure anonymization and assess risk
137
+ - `--lang ja`: issue a Japanese receipt (default is English; JSON output is also available)
138
+ - `--in-format` / `--out-format`: explicitly set the input/output format (csv, json, xml)
139
+ - `sieve-crowd keygen -o key.txt`: generate a cryptographic key for pseudonymization
140
+
141
+ Exit codes:
142
+
143
+ - `0`: success
144
+ - `1`: I/O error
145
+ - `2`: configuration error
146
+ - `3`: refused (the target anonymization level could not be reached; no output written)
147
+
148
+ ---
149
+
150
+ ## πŸ”¬ Tests
151
+
152
+ ```bash
153
+ python -m unittest discover -s tests -v
154
+ ```
155
+
156
+ All test suites run with no external dependencies (standard library only).
157
+
158
+ ---
159
+
160
+ ## ⚠️ Known limitations and caveats
161
+
162
+ 1. **Model assumption** β€” Sieve Crowd protects against an attacker who knows the attributes you declared as `qi` (quasi-identifiers). It is not a universal shield against re-identification through auxiliary information you did not anticipate.
163
+ 2. **Not a substitute for legal or privacy review** β€” It does not guarantee legal or regulatory compliance under privacy laws or guidelines; it is a tool that supports such review technically.
164
+ 3. **Excel is not supported** β€” To keep the dependency-free principle, the binary `.xlsx` format is out of scope (use CSV / JSON / XML).
165
+ 4. **Row shuffling** β€” To minimize re-identification risk, the output row order is deterministically shuffled.
166
+
167
+ ---
168
+
169
+ ## πŸ“„ License
170
+
171
+ MIT License. See [LICENSE](LICENSE) for details.
172
+
173
+ ---
174
+
175
+ ## πŸ‘€ Author
176
+
177
+ * **Kai IWASAKI**
@@ -0,0 +1,19 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "sieve-crowd"
7
+ version = "0.2.0"
8
+ description = "Make every record one of a crowd, and measure it: deterministic, offline k-anonymity for CSV, JSON and XML tables."
9
+ readme = "README.md"
10
+ requires-python = ">=3.10"
11
+ license = { text = "MIT" }
12
+ authors = [{ name = "Kai IWASAKI" }]
13
+ dependencies = []
14
+
15
+ [project.scripts]
16
+ sieve-crowd = "sieve_crowd.cli:main"
17
+
18
+ [tool.setuptools]
19
+ packages = ["sieve_crowd"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,16 @@
1
+ """Sieve Crowd: make every record one of a crowd, and measure it.
2
+
3
+ Deterministic, offline, zero-dependency k-anonymity (with optional l-diversity)
4
+ for CSV, JSON and XML tables.
5
+ """
6
+
7
+ __version__ = "0.2.0"
8
+
9
+ from .config import Config, parse_config, parse_key # noqa: E402
10
+ from .engine import Result, anonymize, inspect_table # noqa: E402
11
+ from .model import ConfigError, InputError, Role # noqa: E402
12
+
13
+ __all__ = [
14
+ "__version__", "anonymize", "inspect_table", "Result", "Config", "parse_config", "parse_key",
15
+ "ConfigError", "InputError", "Role",
16
+ ]
@@ -0,0 +1,3 @@
1
+ from .cli import main
2
+
3
+ raise SystemExit(main())
@@ -0,0 +1,206 @@
1
+ """Command line: inspect / keygen / run. Cell values never appear in messages or logs."""
2
+ from __future__ import annotations
3
+
4
+ import argparse
5
+ import json
6
+ import os
7
+ import secrets
8
+ import sys
9
+ import tempfile
10
+ from typing import Optional, Sequence
11
+
12
+ from . import __version__
13
+ from .config import parse_config, parse_key
14
+ from .engine import anonymize, inspect_table
15
+ from .hints import hint
16
+ from .model import ConfigError, InputError
17
+ from .tableio import FORMATS, detect_format, read_any, write_any
18
+
19
+ EXIT_OK, EXIT_IO, EXIT_CONFIG, EXIT_REFUSED = 0, 1, 2, 3
20
+
21
+
22
+ def _atomic_write(path: str, data: bytes, private: bool = False) -> None:
23
+ directory = os.path.dirname(os.path.abspath(path))
24
+ fd, tmp = tempfile.mkstemp(dir=directory, prefix=".sieve-crowd-")
25
+ try:
26
+ with os.fdopen(fd, "wb") as fh:
27
+ fh.write(data)
28
+ if private:
29
+ os.chmod(tmp, 0o600)
30
+ os.replace(tmp, path)
31
+ except BaseException:
32
+ try:
33
+ os.unlink(tmp)
34
+ except OSError:
35
+ pass
36
+ raise
37
+
38
+
39
+ def _parser() -> argparse.ArgumentParser:
40
+ p = argparse.ArgumentParser(prog="sieve-crowd",
41
+ description="Make every record one of a crowd of at least k, and measure it.")
42
+ p.add_argument("--version", action="version", version=f"sieve-crowd {__version__}")
43
+ sub = p.add_subparsers(dest="cmd", required=True)
44
+
45
+ i = sub.add_parser("inspect", help="describe the columns of a table (csv, json or xml) and write a config template")
46
+ i.add_argument("input")
47
+ i.add_argument("--delimiter", help="CSV only")
48
+ i.add_argument("--in-format", choices=FORMATS,
49
+ help="force the input format (default: from the file extension)")
50
+ i.add_argument("--lang", choices=["en", "ja"], default="en")
51
+ i.add_argument("--config-out", metavar="FILE", help="write a template with role '?' for every column")
52
+ i.add_argument("--force", action="store_true")
53
+
54
+ k = sub.add_parser("keygen", help="create a key for pseudonyms")
55
+ k.add_argument("-o", "--output", required=True)
56
+ k.add_argument("--force", action="store_true")
57
+
58
+ r = sub.add_parser("run", help="anonymize a table (csv, json or xml)")
59
+ r.add_argument("input")
60
+ r.add_argument("-c", "--config", required=True)
61
+ r.add_argument("-o", "--output")
62
+ r.add_argument("--dry-run", action="store_true", help="measure only; write no table")
63
+ r.add_argument("--delimiter", help="CSV only")
64
+ r.add_argument("--in-format", choices=FORMATS,
65
+ help="force the input format (default: from the file extension)")
66
+ r.add_argument("--out-format", choices=FORMATS,
67
+ help="force the output format (default: from the output extension)")
68
+ r.add_argument("--key-file")
69
+ r.add_argument("--k", type=int)
70
+ r.add_argument("--l", type=int)
71
+ r.add_argument("--max-suppression", type=float)
72
+ r.add_argument("--keep-order", action="store_true", help="relaxation: do not shuffle the rows (recorded)")
73
+ r.add_argument("--shuffle-seed", type=int, default=0)
74
+ r.add_argument("--lang", choices=["en", "ja"], default="en")
75
+ r.add_argument("--max-rows", type=int, default=200_000)
76
+ r.add_argument("--max-evals", type=int, default=20_000)
77
+ r.add_argument("--report-json", metavar="FILE")
78
+ r.add_argument("--quiet", action="store_true")
79
+ r.add_argument("--force", action="store_true")
80
+ return p
81
+
82
+
83
+ def _read(path: str) -> bytes:
84
+ if path == "-":
85
+ return sys.stdin.buffer.read()
86
+ with open(path, "rb") as fh:
87
+ return fh.read()
88
+
89
+
90
+ def _resolve_format(path: str, explicit: Optional[str], *, dash_default: str) -> str:
91
+ if explicit:
92
+ return explicit
93
+ if path == "-":
94
+ return dash_default
95
+ return detect_format(os.path.splitext(path)[1])
96
+
97
+
98
+ def _delimiter(given: Optional[str], sides: set[str]) -> Optional[str]:
99
+ if given is None:
100
+ return None
101
+ if sides - {"csv"}:
102
+ raise InputError("--delimiter applies to csv only (the table is json/xml)")
103
+ return given
104
+
105
+
106
+ def _cmd_inspect(a: argparse.Namespace) -> int:
107
+ try:
108
+ in_fmt = _resolve_format(a.input, a.in_format, dash_default="csv")
109
+ delim = _delimiter(a.delimiter, {in_fmt})
110
+ header, rows = read_any(_read(a.input), in_fmt, delim)
111
+ except InputError as exc:
112
+ print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
113
+ except OSError as exc:
114
+ print(f"error: cannot read the input ({exc.strerror})", file=sys.stderr); return EXIT_IO
115
+ info = inspect_table(header, rows)
116
+ en = a.lang == "en"
117
+ print(f"records: {len(rows)}" if en else f"レコード数: {len(rows)}")
118
+ hints = {}
119
+ for c in info:
120
+ role, text_en, text_ja = hint(c["name"], c, len(rows))
121
+ hints[c["name"]] = (role, text_en if en else text_ja)
122
+ shown = f" {c['name']}: {c['distinct']} distinct, {c['empty']} empty, {c['unique_ratio']:.0%} unique values, looks like {c['shape']}" \
123
+ if en else f" {c['name']}: η•°γͺγ‚‹ε€€ {c['distinct']}、空 {c['empty']}、一意γͺε€€ {c['unique_ratio']:.0%}、归は {c['shape']}"
124
+ print(shown)
125
+ if role:
126
+ print(f" -> {hints[c['name']][1]}")
127
+ if a.config_out:
128
+ if os.path.exists(a.config_out) and not a.force:
129
+ print("error: the config file exists (use --force)", file=sys.stderr); return EXIT_IO
130
+ template = {"k": 5, "l": 1, "max_suppression": 0.05,
131
+ "columns": {c["name"]: ({"role": "?", "_hint": hints[c["name"]][1]} if hints[c["name"]][0]
132
+ else {"role": "?"}) for c in info}}
133
+ try:
134
+ _atomic_write(a.config_out, (json.dumps(template, ensure_ascii=False, indent=2) + "\n").encode())
135
+ except OSError as exc:
136
+ print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
137
+ return EXIT_OK
138
+
139
+
140
+ def _cmd_keygen(a: argparse.Namespace) -> int:
141
+ if os.path.exists(a.output) and not a.force:
142
+ print("error: the key file exists (use --force)", file=sys.stderr); return EXIT_IO
143
+ try:
144
+ _atomic_write(a.output, (secrets.token_hex(32) + "\n").encode(), private=True)
145
+ except OSError as exc:
146
+ print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
147
+ return EXIT_OK
148
+
149
+
150
+ def _cmd_run(a: argparse.Namespace) -> int:
151
+ if not a.dry_run and not a.output:
152
+ print("error: give -o/--output or --dry-run", file=sys.stderr); return EXIT_CONFIG
153
+ if a.output and a.input != "-" and os.path.realpath(a.output) == os.path.realpath(a.input):
154
+ print("error: the output must not be the input file", file=sys.stderr); return EXIT_CONFIG
155
+ if a.output and os.path.exists(a.output) and not a.force:
156
+ print("error: output exists (use --force to replace it)", file=sys.stderr); return EXIT_IO
157
+ try:
158
+ in_fmt = _resolve_format(a.input, a.in_format, dash_default="csv")
159
+ out_fmt = _resolve_format(a.output, a.out_format, dash_default="csv") if a.output else None
160
+ delim = _delimiter(a.delimiter, {in_fmt} | (set() if out_fmt is None else {out_fmt}))
161
+ except InputError as exc:
162
+ print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
163
+ try:
164
+ with open(a.config, "rb") as fh:
165
+ raw = json.loads(fh.read().decode("utf-8-sig"))
166
+ key = parse_key(open(a.key_file, encoding="utf-8").read()) if a.key_file else None
167
+ cfg = parse_config(raw, k=a.k, l=a.l, max_suppression=a.max_suppression,
168
+ shuffle=not a.keep_order, shuffle_seed=a.shuffle_seed, lang=a.lang,
169
+ max_rows=a.max_rows, max_evals=a.max_evals, key=key)
170
+ except ConfigError as exc:
171
+ print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
172
+ except (json.JSONDecodeError, UnicodeDecodeError):
173
+ print("error: the configuration is not valid JSON", file=sys.stderr); return EXIT_CONFIG
174
+ except OSError as exc:
175
+ print(f"error: cannot read the configuration or key ({exc.strerror})", file=sys.stderr); return EXIT_IO
176
+ try:
177
+ header, rows = read_any(_read(a.input), in_fmt, delim)
178
+ except InputError as exc:
179
+ print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
180
+ except OSError as exc:
181
+ print(f"error: cannot read the input ({exc.strerror})", file=sys.stderr); return EXIT_IO
182
+ try:
183
+ result = anonymize(header, rows, cfg, io=(in_fmt, out_fmt))
184
+ except ConfigError as exc:
185
+ print(f"error: {exc}", file=sys.stderr); return EXIT_CONFIG
186
+ if not a.quiet:
187
+ print(result.report.text(cfg.lang), file=sys.stderr)
188
+ try:
189
+ if a.report_json:
190
+ _atomic_write(a.report_json, (result.report.to_json() + "\n").encode("utf-8"))
191
+ if result.status == "refused":
192
+ return EXIT_REFUSED
193
+ if not a.dry_run:
194
+ data = write_any(result.header, result.rows, out_fmt, delim)
195
+ if a.output == "-":
196
+ sys.stdout.buffer.write(data); sys.stdout.buffer.flush()
197
+ else:
198
+ _atomic_write(a.output, data)
199
+ except OSError as exc:
200
+ print(f"error: cannot write ({exc.strerror})", file=sys.stderr); return EXIT_IO
201
+ return EXIT_OK
202
+
203
+
204
+ def main(argv: Optional[Sequence[str]] = None) -> int:
205
+ a = _parser().parse_args(argv)
206
+ return {"inspect": _cmd_inspect, "keygen": _cmd_keygen, "run": _cmd_run}[a.cmd](a)