wowool-anonymizer 2.1.1.dev13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wowool/anonymizer/__init__.py +1 -0
- wowool/anonymizer/core/anonymizer.py +164 -0
- wowool/anonymizer/core/anonymizer_config.py +227 -0
- wowool/anonymizer/core/app_id.py +1 -0
- wowool/anonymizer/core/byte_writer.py +98 -0
- wowool/anonymizer/core/default_writer.py +244 -0
- wowool/anonymizer/core/defines.py +10 -0
- wowool/anonymizer/core/writer_interface.py +27 -0
- wowool/anonymizer/writers/factory/writer.py +44 -0
- wowool/anonymizer/writers/pdf/writer.py +605 -0
- wowool/anonymizer/writers/text/writer.py +39 -0
- wowool/apps/anonymizer/__init__.py +2 -0
- wowool/apps/anonymizer/config.py +4 -0
- wowool/tools/anonymizer/__main__.py +123 -0
- wowool/tools/anonymizer/writer_config.py +10 -0
- wowool/tools/pdf_anonymizer/__main__.py +45 -0
- wowool/tools/pdf_anonymizer/pdf_wow_anonymizer.json +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/METADATA +19 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/RECORD +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/WHEEL +5 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/entry_points.txt +2 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,244 @@
|
|
|
1
|
+
from wowool.annotation import Concept
|
|
2
|
+
from faker import Faker
|
|
3
|
+
from functools import wraps
|
|
4
|
+
from random import randint
|
|
5
|
+
import copy
|
|
6
|
+
from wowool.anonymizer.core.writer_interface import WriterInterface
|
|
7
|
+
from wowool.anonymizer.core.defines import (
|
|
8
|
+
KW_URI,
|
|
9
|
+
KW_ANONYMIZED,
|
|
10
|
+
KW_TEXT,
|
|
11
|
+
KW_BEGIN_OFFSET,
|
|
12
|
+
KW_END_OFFSET,
|
|
13
|
+
KW_BYTE_BEGIN_OFFSET,
|
|
14
|
+
KW_BYTE_END_OFFSET,
|
|
15
|
+
)
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def check_faker(fn):
|
|
19
|
+
@wraps(fn)
|
|
20
|
+
def wrapper(self, concept: Concept, *args, **kwargs):
|
|
21
|
+
if not self.fake:
|
|
22
|
+
self.fake = Faker(self.fake_locale)
|
|
23
|
+
return fn(self, concept, *args, **kwargs)
|
|
24
|
+
|
|
25
|
+
return wrapper
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class DefaultWriter(WriterInterface):
|
|
29
|
+
"""
|
|
30
|
+
This object will rewrite the original document.
|
|
31
|
+
|
|
32
|
+
:param dummies: A dictionary that has a list of fake names for a given uri. Example: { "Person" : ["Hulk","Flash"] }
|
|
33
|
+
:type dummies: dict( uri: list(names) )
|
|
34
|
+
:param formatters: Format how an entity should be displayed in the document. Example: { "Person" : "{anonymized_literal}" }
|
|
35
|
+
:type formatters: dict(uri, fstring format), internal variables that can be used. anonymized_literal, uri, nr, literal
|
|
36
|
+
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(
|
|
40
|
+
self,
|
|
41
|
+
pseudonyms={},
|
|
42
|
+
formatters=None,
|
|
43
|
+
fake_locale: list[str] | None = None,
|
|
44
|
+
):
|
|
45
|
+
self.pseudonyms = copy.deepcopy(pseudonyms)
|
|
46
|
+
self.formatters = formatters
|
|
47
|
+
self.clear()
|
|
48
|
+
self.reset_document_data()
|
|
49
|
+
self.input_data_ = None
|
|
50
|
+
self.locations = []
|
|
51
|
+
self.fake = None
|
|
52
|
+
self.fake_locale = ["en_US", "en_GB"] if not fake_locale else fake_locale
|
|
53
|
+
|
|
54
|
+
@check_faker
|
|
55
|
+
def random_instance(self, concept: Concept):
|
|
56
|
+
retval = ""
|
|
57
|
+
|
|
58
|
+
if concept.canonical not in self.known_entries:
|
|
59
|
+
match concept.uri:
|
|
60
|
+
case "Person":
|
|
61
|
+
gender = concept.gender
|
|
62
|
+
match gender:
|
|
63
|
+
case "male":
|
|
64
|
+
retval = self.fake.name_male()
|
|
65
|
+
case "female":
|
|
66
|
+
retval = self.fake.name_female()
|
|
67
|
+
case _:
|
|
68
|
+
retval = self.fake.name()
|
|
69
|
+
case "Company":
|
|
70
|
+
retval = self.fake.company()
|
|
71
|
+
case "Position":
|
|
72
|
+
retval = self.fake.job()
|
|
73
|
+
case "Email":
|
|
74
|
+
retval = self.fake.free_email()
|
|
75
|
+
case "Country":
|
|
76
|
+
retval = self.fake.country()
|
|
77
|
+
case "City":
|
|
78
|
+
retval = self.fake.city()
|
|
79
|
+
case "Address":
|
|
80
|
+
retval = self.fake.address()
|
|
81
|
+
case _:
|
|
82
|
+
retval = "###"
|
|
83
|
+
self.known_entries[concept.canonical] = {"uri": retval, "nr": 0}
|
|
84
|
+
return retval
|
|
85
|
+
else:
|
|
86
|
+
return self.known_entries[concept.canonical]["uri"]
|
|
87
|
+
|
|
88
|
+
def reset_document_data(self):
|
|
89
|
+
"""clear the known_entries of the writer, which means that previous entries will not be the same in the following document"""
|
|
90
|
+
self.counter, self.known_entries = {}, {}
|
|
91
|
+
|
|
92
|
+
def set_input_data(self, input_data: str):
|
|
93
|
+
"""set the input data of the writer"""
|
|
94
|
+
self.input_data_ = input_data
|
|
95
|
+
|
|
96
|
+
@property
|
|
97
|
+
def input_data(self):
|
|
98
|
+
"""return the input data of the writer"""
|
|
99
|
+
return self.input_data_
|
|
100
|
+
|
|
101
|
+
def clear(self):
|
|
102
|
+
"""clear the output data of the writer"""
|
|
103
|
+
self.out = ""
|
|
104
|
+
self.locations = []
|
|
105
|
+
self.input_data_ = None
|
|
106
|
+
|
|
107
|
+
def _add_literal_data(self, concept: Concept, literal):
|
|
108
|
+
retval = self.format_string(concept)
|
|
109
|
+
if retval:
|
|
110
|
+
self.known_entries[literal] = {"uri": retval, "nr": 0}
|
|
111
|
+
else:
|
|
112
|
+
uri = concept.uri
|
|
113
|
+
self.update_uri_counter(uri)
|
|
114
|
+
self.known_entries[literal] = {
|
|
115
|
+
"uri": f"{uri}_{self.counter[uri]}",
|
|
116
|
+
"nr": self.counter[uri],
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
def format_string(
|
|
120
|
+
self, concept: Concept, anonymized_literal_candidate: str | None = None
|
|
121
|
+
):
|
|
122
|
+
canonical = concept.canonical
|
|
123
|
+
uri = concept.uri
|
|
124
|
+
literal = concept.literal # noqa
|
|
125
|
+
kw_literal = canonical
|
|
126
|
+
concept_format_string = concept.attributes.get("formatter", None)
|
|
127
|
+
if self.formatters or concept_format_string:
|
|
128
|
+
nr = "not_set"
|
|
129
|
+
literal_data = self.known_entries.get(kw_literal, None)
|
|
130
|
+
if literal_data is not None:
|
|
131
|
+
nr = literal_data.get("nr", "not_set")
|
|
132
|
+
anonymized_literal = literal_data.get(
|
|
133
|
+
"uri", anonymized_literal_candidate
|
|
134
|
+
)
|
|
135
|
+
else:
|
|
136
|
+
nr = self.counter.get(uri, 1)
|
|
137
|
+
anonymized_literal = anonymized_literal_candidate
|
|
138
|
+
|
|
139
|
+
if anonymized_literal is None:
|
|
140
|
+
anonymized_literal = f"{uri}_{nr}"
|
|
141
|
+
anonymized = anonymized_literal # noqa used in f-string
|
|
142
|
+
|
|
143
|
+
if uri not in self.formatters and "default" in self.formatters:
|
|
144
|
+
mapping_kw = "default"
|
|
145
|
+
else:
|
|
146
|
+
mapping_kw = uri
|
|
147
|
+
|
|
148
|
+
if mapping_kw in self.formatters:
|
|
149
|
+
if "fake" in self.formatters[mapping_kw]:
|
|
150
|
+
fake = self.random_instance(concept) # noqa
|
|
151
|
+
|
|
152
|
+
anonymized_literal = eval('f"""' + self.formatters[mapping_kw] + '"""')
|
|
153
|
+
return anonymized_literal
|
|
154
|
+
elif concept_format_string:
|
|
155
|
+
anonymized_literal = eval('f"""' + anonymized_literal + '"""')
|
|
156
|
+
return anonymized_literal
|
|
157
|
+
else:
|
|
158
|
+
return anonymized_literal
|
|
159
|
+
|
|
160
|
+
def update_uri_counter(self, uri: str):
|
|
161
|
+
"""update the counter for the given uri"""
|
|
162
|
+
if uri not in self.counter:
|
|
163
|
+
self.counter[uri] = 1
|
|
164
|
+
else:
|
|
165
|
+
self.counter[uri] += 1
|
|
166
|
+
|
|
167
|
+
def anonymize(self, concept: Concept):
|
|
168
|
+
"""
|
|
169
|
+
return a formated str or anonymized literal for the given literal with the given uri.
|
|
170
|
+
|
|
171
|
+
:param uri: uri for the given literal
|
|
172
|
+
:type uri: str
|
|
173
|
+
:param literal: literal that needs to be anonymized
|
|
174
|
+
:type literal: str
|
|
175
|
+
|
|
176
|
+
"""
|
|
177
|
+
uri = concept.uri
|
|
178
|
+
kw_literal = concept.canonical
|
|
179
|
+
if kw_literal not in self.known_entries:
|
|
180
|
+
if (
|
|
181
|
+
self.pseudonyms
|
|
182
|
+
and uri in self.pseudonyms
|
|
183
|
+
and len(self.pseudonyms[uri]) > 0
|
|
184
|
+
):
|
|
185
|
+
candidates = self.pseudonyms[uri]
|
|
186
|
+
if candidates:
|
|
187
|
+
self.update_uri_counter(uri)
|
|
188
|
+
|
|
189
|
+
item_index = randint(0, len(candidates) - 1)
|
|
190
|
+
retval = self.format_string(
|
|
191
|
+
concept, anonymized_literal_candidate=candidates[item_index]
|
|
192
|
+
)
|
|
193
|
+
anonymized_literal_ = retval if retval else candidates[item_index]
|
|
194
|
+
|
|
195
|
+
self.known_entries[kw_literal] = {
|
|
196
|
+
"uri": anonymized_literal_,
|
|
197
|
+
"nr": self.counter[uri],
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
# remove this one from the list to avoid duplicates
|
|
201
|
+
del candidates[item_index]
|
|
202
|
+
else:
|
|
203
|
+
self._add_literal_data(concept, kw_literal)
|
|
204
|
+
else:
|
|
205
|
+
self._add_literal_data(concept, kw_literal)
|
|
206
|
+
anonymized_literal = self.known_entries[kw_literal]["uri"]
|
|
207
|
+
return anonymized_literal
|
|
208
|
+
|
|
209
|
+
def write(self, bytes_data: str):
|
|
210
|
+
"""write the given data to the output result"""
|
|
211
|
+
self.out += bytes_data
|
|
212
|
+
|
|
213
|
+
def write_location(self, bytes_data: str, concept: Concept):
|
|
214
|
+
"""write the given data to the output result"""
|
|
215
|
+
begin_offset = len(self.out)
|
|
216
|
+
self.out += bytes_data
|
|
217
|
+
end_offset = len(self.out)
|
|
218
|
+
self.locations.append(
|
|
219
|
+
{
|
|
220
|
+
KW_BEGIN_OFFSET: begin_offset,
|
|
221
|
+
KW_END_OFFSET: end_offset,
|
|
222
|
+
KW_TEXT: concept.canonical,
|
|
223
|
+
KW_URI: concept.uri,
|
|
224
|
+
KW_ANONYMIZED: bytes_data,
|
|
225
|
+
KW_BYTE_BEGIN_OFFSET: concept.begin_offset,
|
|
226
|
+
KW_BYTE_END_OFFSET: concept.end_offset,
|
|
227
|
+
}
|
|
228
|
+
)
|
|
229
|
+
|
|
230
|
+
def write_substr(self, start: int, end: int | None = None):
|
|
231
|
+
"""write the given data to the output result"""
|
|
232
|
+
assert self.input_data_ is not None, "input data is not set"
|
|
233
|
+
if end is None:
|
|
234
|
+
self.out += self.input_data_[start:]
|
|
235
|
+
else:
|
|
236
|
+
self.out += self.input_data_[start:end]
|
|
237
|
+
|
|
238
|
+
def get_value(self):
|
|
239
|
+
"""return output result"""
|
|
240
|
+
return {"text": self.out, "locations": self.locations}
|
|
241
|
+
|
|
242
|
+
def get_bytes(self):
|
|
243
|
+
"""return output result"""
|
|
244
|
+
return self.out.encode()
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
KW_URI = "uri"
|
|
2
|
+
KW_ANONYMIZED = "anonymized"
|
|
3
|
+
KW_TEXT = "text"
|
|
4
|
+
KW_BEGIN_OFFSET = "begin_offset"
|
|
5
|
+
KW_END_OFFSET = "end_offset"
|
|
6
|
+
KW_BYTE_BEGIN_OFFSET = "byte_begin_offset"
|
|
7
|
+
KW_BYTE_END_OFFSET = "byte_end_offset"
|
|
8
|
+
KW_ANONYMIZED_LITERAL = "anonymized_literal"
|
|
9
|
+
KW_LITERAL = "literal"
|
|
10
|
+
KW_CANONICAL = "canonical"
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
from typing import Protocol, runtime_checkable, Any
|
|
2
|
+
from wowool.annotation import Concept
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
@runtime_checkable
|
|
6
|
+
class WriterInterface(Protocol):
|
|
7
|
+
|
|
8
|
+
def anonymize(self, concept: Concept):
|
|
9
|
+
"""
|
|
10
|
+
return a formatted str or anonymized literal for the given literal with the given uri.
|
|
11
|
+
|
|
12
|
+
:param concept: concept that needs to be anonymized
|
|
13
|
+
:type concept: Concept
|
|
14
|
+
"""
|
|
15
|
+
return "***"
|
|
16
|
+
|
|
17
|
+
def write(self, bytes_data: str):
|
|
18
|
+
"""write the given data to the output result"""
|
|
19
|
+
pass
|
|
20
|
+
|
|
21
|
+
def clear(self):
|
|
22
|
+
"""clear the output data of the writer"""
|
|
23
|
+
pass
|
|
24
|
+
|
|
25
|
+
def get_value(self) -> dict[str, Any]:
|
|
26
|
+
"""return the output data of the writer"""
|
|
27
|
+
pass
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
from wowool.document import DocumentInterface
|
|
2
|
+
from wowool.sdk import Pipeline
|
|
3
|
+
from wowool.tools.anonymizer.writer_config import WriterConfig
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class Writer:
|
|
7
|
+
"""
|
|
8
|
+
Base class for all writers.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
def __init__(self, writer_config: WriterConfig):
|
|
12
|
+
self.writer_config = writer_config
|
|
13
|
+
self.anonymizers = {}
|
|
14
|
+
|
|
15
|
+
def get_anonymizer(self, mime_type):
|
|
16
|
+
"""
|
|
17
|
+
Get the anonymizers for the writer.
|
|
18
|
+
"""
|
|
19
|
+
if mime_type not in self.anonymizers:
|
|
20
|
+
if mime_type == "text/plain":
|
|
21
|
+
from wowool.anonymizer.writers.text.writer import Writer as TextWriter
|
|
22
|
+
|
|
23
|
+
self.anonymizers[mime_type] = TextWriter(self.writer_config)
|
|
24
|
+
return self.anonymizers[mime_type]
|
|
25
|
+
elif mime_type == "application/pdf":
|
|
26
|
+
from wowool.anonymizer.writers.pdf.writer import Writer as PdfWriter
|
|
27
|
+
|
|
28
|
+
self.anonymizers[mime_type] = PdfWriter(self.writer_config)
|
|
29
|
+
return self.anonymizers[mime_type]
|
|
30
|
+
else:
|
|
31
|
+
return self.anonymizers[mime_type]
|
|
32
|
+
|
|
33
|
+
def __call__(self, document: DocumentInterface, pipeline: Pipeline):
|
|
34
|
+
"""
|
|
35
|
+
Write the document to the specified path.
|
|
36
|
+
"""
|
|
37
|
+
if not isinstance(document, DocumentInterface):
|
|
38
|
+
raise TypeError("document must be an instance of DocumentInterface")
|
|
39
|
+
if not isinstance(pipeline, Pipeline):
|
|
40
|
+
raise TypeError("pipeline must be an instance of Pipeline")
|
|
41
|
+
if anonymizer_ := self.get_anonymizer(document.mime_type):
|
|
42
|
+
return anonymizer_(document, pipeline)
|
|
43
|
+
else:
|
|
44
|
+
raise NotImplementedError(f"Writer unknown data type: {document.mime_type}")
|