wowool-anonymizer 2.1.1.dev13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wowool/anonymizer/__init__.py +1 -0
- wowool/anonymizer/core/anonymizer.py +164 -0
- wowool/anonymizer/core/anonymizer_config.py +227 -0
- wowool/anonymizer/core/app_id.py +1 -0
- wowool/anonymizer/core/byte_writer.py +98 -0
- wowool/anonymizer/core/default_writer.py +244 -0
- wowool/anonymizer/core/defines.py +10 -0
- wowool/anonymizer/core/writer_interface.py +27 -0
- wowool/anonymizer/writers/factory/writer.py +44 -0
- wowool/anonymizer/writers/pdf/writer.py +605 -0
- wowool/anonymizer/writers/text/writer.py +39 -0
- wowool/apps/anonymizer/__init__.py +2 -0
- wowool/apps/anonymizer/config.py +4 -0
- wowool/tools/anonymizer/__main__.py +123 -0
- wowool/tools/anonymizer/writer_config.py +10 -0
- wowool/tools/pdf_anonymizer/__main__.py +45 -0
- wowool/tools/pdf_anonymizer/pdf_wow_anonymizer.json +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/METADATA +19 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/RECORD +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/WHEEL +5 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/entry_points.txt +2 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
#!/usr/bin/python3
|
|
2
|
+
# ex : python3 -m wowool.anonymizer -f ~/dev/csv2_test/net.csv
|
|
3
|
+
|
|
4
|
+
from jsonargparse import ArgumentParser, ActionConfigFile, set_loader
|
|
5
|
+
from argparse import RawDescriptionHelpFormatter
|
|
6
|
+
import sys
|
|
7
|
+
from wowool.document.document_interface import DocumentInterface
|
|
8
|
+
from wowool.native.core.pipeline import Pipeline
|
|
9
|
+
from wowool.anonymizer.core.anonymizer import Anonymizer, DefaultWriter
|
|
10
|
+
from wowool.anonymizer.core.anonymizer_config import DEFAULT_PSEUDONYMS
|
|
11
|
+
from wowool.utility import clean_up_empty_keywords
|
|
12
|
+
from wowool.document import Document
|
|
13
|
+
from wowool.error import Error
|
|
14
|
+
from sys import stderr
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from wowool.utility.apps.yml_include import custom_yaml_load
|
|
17
|
+
from wowool.tools.anonymizer.writer_config import WriterConfig
|
|
18
|
+
from wowool.anonymizer.writers.factory.writer import Writer
|
|
19
|
+
|
|
20
|
+
set_loader("yaml_custom", custom_yaml_load)
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def parse_arguments(argv):
|
|
24
|
+
"""
|
|
25
|
+
The anonymizer is an EyeOnText tool that allows you to obscure data from documents
|
|
26
|
+
|
|
27
|
+
Example: anonymizer -f ~/corpus/english --pipeline english,entity --suffix '.out'
|
|
28
|
+
"""
|
|
29
|
+
parser = ArgumentParser(
|
|
30
|
+
prog="./anonymizer",
|
|
31
|
+
fromfile_prefix_chars="@",
|
|
32
|
+
formatter_class=RawDescriptionHelpFormatter,
|
|
33
|
+
description=parse_arguments.__doc__,
|
|
34
|
+
)
|
|
35
|
+
|
|
36
|
+
parser.add_argument("-f", "--file", help="folder or file", required=True)
|
|
37
|
+
parser.add_argument(
|
|
38
|
+
"-p",
|
|
39
|
+
"--pipeline",
|
|
40
|
+
help="domains used to anonymize the document, without the anonymizer.app",
|
|
41
|
+
required=True,
|
|
42
|
+
)
|
|
43
|
+
parser.add_argument(
|
|
44
|
+
"-a", "--annotations", help="The entities that you want to anonymize"
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
parser.add_argument(
|
|
48
|
+
"--formatters",
|
|
49
|
+
help="description on how to format the output, dict[str,str]",
|
|
50
|
+
default=None,
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
parser.add_argument(
|
|
54
|
+
"--pseudonyms",
|
|
55
|
+
help="dictionary of with list of pseudonyms, dict[str,list[str]]",
|
|
56
|
+
default=None,
|
|
57
|
+
)
|
|
58
|
+
|
|
59
|
+
parser.add_argument("--suffix", help="suffix of the output files.")
|
|
60
|
+
parser.add_argument("-c", "--config", action=ActionConfigFile)
|
|
61
|
+
return parser.parse_args(argv)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def get_anonymizer(anonymizers, document: DocumentInterface, config, writer):
|
|
65
|
+
|
|
66
|
+
if document.mime_type not in anonymizers:
|
|
67
|
+
print(document.mime_type)
|
|
68
|
+
anonymizers[document.mime_type] = Anonymizer(config.annotations, writer)
|
|
69
|
+
return anonymizers[document.mime_type]
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get_writer(writers, document: DocumentInterface, config):
|
|
73
|
+
|
|
74
|
+
if document.mime_type not in writers:
|
|
75
|
+
print(document.mime_type)
|
|
76
|
+
writer = DefaultWriter(config.pseudonyms, config.formatters)
|
|
77
|
+
writers[document.mime_type] = writer
|
|
78
|
+
return writers[document.mime_type]
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def anonymizer(
|
|
82
|
+
file,
|
|
83
|
+
writer_config: WriterConfig,
|
|
84
|
+
pipeline,
|
|
85
|
+
):
|
|
86
|
+
|
|
87
|
+
ip_collection = [fn for fn in Document.glob(Path(file), raw=True)]
|
|
88
|
+
try:
|
|
89
|
+
pipeline_ = Pipeline(pipeline)
|
|
90
|
+
|
|
91
|
+
except Error as ex:
|
|
92
|
+
print(ex)
|
|
93
|
+
exit(-1)
|
|
94
|
+
|
|
95
|
+
writer = Writer(writer_config)
|
|
96
|
+
for document in ip_collection:
|
|
97
|
+
print(f"processing:{document.id} ...", file=stderr)
|
|
98
|
+
output_fn = writer(document, pipeline_)
|
|
99
|
+
print(f"file:{document.id} -> {output_fn}", file=stderr)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def main():
|
|
103
|
+
args = dict(parse_arguments(sys.argv[1:])._get_kwargs())
|
|
104
|
+
del args["config"]
|
|
105
|
+
clean_up_empty_keywords(args)
|
|
106
|
+
writer_config = WriterConfig(annotations=args["annotations"])
|
|
107
|
+
if "formatters" in args:
|
|
108
|
+
writer_config.formatters = args["formatters"]
|
|
109
|
+
del args["formatters"]
|
|
110
|
+
|
|
111
|
+
if "pseudonyms" in args:
|
|
112
|
+
writer_config.pseudonyms = args["pseudonyms"]
|
|
113
|
+
del args["pseudonyms"]
|
|
114
|
+
else:
|
|
115
|
+
writer_config.pseudonyms = DEFAULT_PSEUDONYMS
|
|
116
|
+
|
|
117
|
+
anonymizer(
|
|
118
|
+
writer_config=writer_config, file=args["file"], pipeline=args["pipeline"]
|
|
119
|
+
)
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
if __name__ == "__main__":
|
|
123
|
+
main()
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
from dataclasses import dataclass, field
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class WriterConfig:
|
|
6
|
+
formatters: dict[str, str] | None = None
|
|
7
|
+
pseudonyms: dict[str, list[str]] | None = None
|
|
8
|
+
fake_locale: list[str] | None = None # ["en_US", "en_GB"]
|
|
9
|
+
annotations: list[str] | None = None
|
|
10
|
+
suffix: str = ".anonymized"
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
#!python
|
|
2
|
+
import sys
|
|
3
|
+
from pathlib import Path
|
|
4
|
+
import json
|
|
5
|
+
from wowool.anonymizer.pdf.transform import anonymize_pdf_document
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
DEFAULT_CONFIG = {
|
|
9
|
+
"pipeline": [
|
|
10
|
+
"swedish",
|
|
11
|
+
"actapublica",
|
|
12
|
+
{
|
|
13
|
+
"name": "anonymizer.app",
|
|
14
|
+
"options": {
|
|
15
|
+
"annotations": ["Person", "City", "PersonalIdentificationNumber"],
|
|
16
|
+
"formatters": {"Person": "#{uri}_{nr}#"},
|
|
17
|
+
},
|
|
18
|
+
},
|
|
19
|
+
]
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def main() -> None:
|
|
24
|
+
args = sys.argv
|
|
25
|
+
if len(args) >= 2:
|
|
26
|
+
input_pdf_path = Path(args[1])
|
|
27
|
+
if len(args) >= 3:
|
|
28
|
+
output_pdf_path = Path(args[2])
|
|
29
|
+
else:
|
|
30
|
+
output_pdf_path = input_pdf_path.with_suffix(".anonymized.pdf")
|
|
31
|
+
application_config_fn = Path(args[0]).with_suffix(".json")
|
|
32
|
+
if application_config_fn.exists():
|
|
33
|
+
with open(application_config_fn, "r") as f:
|
|
34
|
+
config = json.load(f)
|
|
35
|
+
else:
|
|
36
|
+
config = DEFAULT_CONFIG
|
|
37
|
+
anonymize_pdf_document(input_pdf_path, config, output_pdf_path)
|
|
38
|
+
else:
|
|
39
|
+
print("Usage: python pdf_wow_anonymizer.py <input_pdf_path> [<output_pdf_path>]")
|
|
40
|
+
sys.exit(1)
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
if __name__ == "__main__":
|
|
44
|
+
|
|
45
|
+
main()
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
{
|
|
2
|
+
"pipeline": [
|
|
3
|
+
"swedish",
|
|
4
|
+
"actapublica",
|
|
5
|
+
{
|
|
6
|
+
"name": "anonymizer.app",
|
|
7
|
+
"options": {
|
|
8
|
+
"annotations": [
|
|
9
|
+
"Person",
|
|
10
|
+
"Address",
|
|
11
|
+
"Street",
|
|
12
|
+
"Email",
|
|
13
|
+
"PersonalIdentificationNumber"
|
|
14
|
+
],
|
|
15
|
+
"formatters": {
|
|
16
|
+
"Person": "#{uri}-{concept.position}-{nr}#",
|
|
17
|
+
"PersonalIdentificationNumber": "#{\"*\"* (len(literal)-3)}{literal[-2:]}"
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
]
|
|
22
|
+
}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: wowool-anonymizer
|
|
3
|
+
Version: 2.1.1.dev13
|
|
4
|
+
Summary: The Wowool Anonymizer Package
|
|
5
|
+
Home-page: https://www.wowool.com/
|
|
6
|
+
Author: Wowool
|
|
7
|
+
Author-email: info@wowool.com
|
|
8
|
+
Requires-Python: >=3.10
|
|
9
|
+
Requires-Dist: Faker==36.1.0
|
|
10
|
+
Requires-Dist: jsonargparse
|
|
11
|
+
Dynamic: author
|
|
12
|
+
Dynamic: author-email
|
|
13
|
+
Dynamic: description
|
|
14
|
+
Dynamic: home-page
|
|
15
|
+
Dynamic: requires-dist
|
|
16
|
+
Dynamic: requires-python
|
|
17
|
+
Dynamic: summary
|
|
18
|
+
|
|
19
|
+
The Wowool Anonymizer Package
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
wowool/anonymizer/__init__.py,sha256=bPzH5GkbfZYji9vlyGFdCbeESLTTPPt5oFu_6Y-Qgjo,86
|
|
2
|
+
wowool/anonymizer/core/anonymizer.py,sha256=lj3lYNMkRxhbjBFzn9RR7jmqAo8fS_mzab5cayW2vn0,5719
|
|
3
|
+
wowool/anonymizer/core/anonymizer_config.py,sha256=5Y7Ks3ZKFby4QKn7RSn9Oqq-On1jeRfmO862u3_zSZw,5672
|
|
4
|
+
wowool/anonymizer/core/app_id.py,sha256=KwszCISeddauOpRUrXo-zRytrXrQjNpkaITfYUgUy3o,29
|
|
5
|
+
wowool/anonymizer/core/byte_writer.py,sha256=-dn_2kWJC6aqoC1AInxV3fZlg9VMwUCCNPyEpSKJLA8,3487
|
|
6
|
+
wowool/anonymizer/core/default_writer.py,sha256=o-8byuyJCghiUy0gNO3MHlF2h9SQ6rBY9XUH_z8gAi0,8717
|
|
7
|
+
wowool/anonymizer/core/defines.py,sha256=JfooBzJWMTHmqWCpmrhCq_pl1s3aIw6sJVRiUhVhubQ,300
|
|
8
|
+
wowool/anonymizer/core/writer_interface.py,sha256=78kM7g4BRTVVBOnKhZz7zRWLYKX-is1vvmYiJ1yZqA4,721
|
|
9
|
+
wowool/anonymizer/writers/factory/writer.py,sha256=eL9REriIbcx7vHY5fXEQGHfM6wvUXlmWKFR-q3Jjyq8,1694
|
|
10
|
+
wowool/anonymizer/writers/pdf/writer.py,sha256=14mB1I_LxBGqUQGl8gBAZIjfDQ4Y8WocrqfPLoDMZH4,21581
|
|
11
|
+
wowool/anonymizer/writers/text/writer.py,sha256=PGUvz5n3AInQWlTX6AEuIu3-2CcZVSImcY5XXxrJ5TU,1437
|
|
12
|
+
wowool/apps/anonymizer/__init__.py,sha256=dVTMxVlUbTMKA0dMLakDQhkS9WeD6KU4Qgx_JZW0aQ8,122
|
|
13
|
+
wowool/apps/anonymizer/config.py,sha256=wiwo6Zkwpjw2m-QAEq86PYeU4knWOg5gXMSzjx8VPz4,420
|
|
14
|
+
wowool/tools/anonymizer/__main__.py,sha256=U76MwpGBiXCIPtA-1xgS-t_vFZOxCfvfabAFF0K-lfc,3841
|
|
15
|
+
wowool/tools/anonymizer/writer_config.py,sha256=bZGDdYPP3GrTIcHPuqjYlRaLtRqqmrACUit96ScB-KU,306
|
|
16
|
+
wowool/tools/pdf_anonymizer/__main__.py,sha256=fT4JD5sI63U6jn1vGSoM8NLBBg_r3_udMbUDpZc_RWM,1205
|
|
17
|
+
wowool/tools/pdf_anonymizer/pdf_wow_anonymizer.json,sha256=XktZr8nc_HJFVxfqaDaNA_dfuD9A_d4-MyUSVNvdAAE,599
|
|
18
|
+
wowool_anonymizer-2.1.1.dev13.dist-info/METADATA,sha256=8a3X40V7pmbAdAizfzYgY5MWNel5BVQIynHvx4J2P4s,441
|
|
19
|
+
wowool_anonymizer-2.1.1.dev13.dist-info/WHEEL,sha256=zaaOINJESkSfm_4HQVc5ssNzHCPXhJm0kEUakpsEHaU,91
|
|
20
|
+
wowool_anonymizer-2.1.1.dev13.dist-info/entry_points.txt,sha256=OlMQfuAbCgw_JdklH2lae2qim0BvLb9V5R2SM6YlBY0,69
|
|
21
|
+
wowool_anonymizer-2.1.1.dev13.dist-info/top_level.txt,sha256=zerqoDN-iHSHwsS1SN2h0q4H6RI1FOVeHW8fNQC9YdU,7
|
|
22
|
+
wowool_anonymizer-2.1.1.dev13.dist-info/RECORD,,
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
wowool
|