wowool-anonymizer 2.1.1.dev13__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,244 @@
1
+ from wowool.annotation import Concept
2
+ from faker import Faker
3
+ from functools import wraps
4
+ from random import randint
5
+ import copy
6
+ from wowool.anonymizer.core.writer_interface import WriterInterface
7
+ from wowool.anonymizer.core.defines import (
8
+ KW_URI,
9
+ KW_ANONYMIZED,
10
+ KW_TEXT,
11
+ KW_BEGIN_OFFSET,
12
+ KW_END_OFFSET,
13
+ KW_BYTE_BEGIN_OFFSET,
14
+ KW_BYTE_END_OFFSET,
15
+ )
16
+
17
+
18
+ def check_faker(fn):
19
+ @wraps(fn)
20
+ def wrapper(self, concept: Concept, *args, **kwargs):
21
+ if not self.fake:
22
+ self.fake = Faker(self.fake_locale)
23
+ return fn(self, concept, *args, **kwargs)
24
+
25
+ return wrapper
26
+
27
+
28
+ class DefaultWriter(WriterInterface):
29
+ """
30
+ This object will rewrite the original document.
31
+
32
+ :param dummies: A dictionary that has a list of fake names for a given uri. Example: { "Person" : ["Hulk","Flash"] }
33
+ :type dummies: dict( uri: list(names) )
34
+ :param formatters: Format how an entity should be displayed in the document. Example: { "Person" : "{anonymized_literal}" }
35
+ :type formatters: dict(uri, fstring format), internal variables that can be used. anonymized_literal, uri, nr, literal
36
+
37
+ """
38
+
39
+ def __init__(
40
+ self,
41
+ pseudonyms={},
42
+ formatters=None,
43
+ fake_locale: list[str] | None = None,
44
+ ):
45
+ self.pseudonyms = copy.deepcopy(pseudonyms)
46
+ self.formatters = formatters
47
+ self.clear()
48
+ self.reset_document_data()
49
+ self.input_data_ = None
50
+ self.locations = []
51
+ self.fake = None
52
+ self.fake_locale = ["en_US", "en_GB"] if not fake_locale else fake_locale
53
+
54
+ @check_faker
55
+ def random_instance(self, concept: Concept):
56
+ retval = ""
57
+
58
+ if concept.canonical not in self.known_entries:
59
+ match concept.uri:
60
+ case "Person":
61
+ gender = concept.gender
62
+ match gender:
63
+ case "male":
64
+ retval = self.fake.name_male()
65
+ case "female":
66
+ retval = self.fake.name_female()
67
+ case _:
68
+ retval = self.fake.name()
69
+ case "Company":
70
+ retval = self.fake.company()
71
+ case "Position":
72
+ retval = self.fake.job()
73
+ case "Email":
74
+ retval = self.fake.free_email()
75
+ case "Country":
76
+ retval = self.fake.country()
77
+ case "City":
78
+ retval = self.fake.city()
79
+ case "Address":
80
+ retval = self.fake.address()
81
+ case _:
82
+ retval = "###"
83
+ self.known_entries[concept.canonical] = {"uri": retval, "nr": 0}
84
+ return retval
85
+ else:
86
+ return self.known_entries[concept.canonical]["uri"]
87
+
88
+ def reset_document_data(self):
89
+ """clear the known_entries of the writer, which means that previous entries will not be the same in the following document"""
90
+ self.counter, self.known_entries = {}, {}
91
+
92
+ def set_input_data(self, input_data: str):
93
+ """set the input data of the writer"""
94
+ self.input_data_ = input_data
95
+
96
+ @property
97
+ def input_data(self):
98
+ """return the input data of the writer"""
99
+ return self.input_data_
100
+
101
+ def clear(self):
102
+ """clear the output data of the writer"""
103
+ self.out = ""
104
+ self.locations = []
105
+ self.input_data_ = None
106
+
107
+ def _add_literal_data(self, concept: Concept, literal):
108
+ retval = self.format_string(concept)
109
+ if retval:
110
+ self.known_entries[literal] = {"uri": retval, "nr": 0}
111
+ else:
112
+ uri = concept.uri
113
+ self.update_uri_counter(uri)
114
+ self.known_entries[literal] = {
115
+ "uri": f"{uri}_{self.counter[uri]}",
116
+ "nr": self.counter[uri],
117
+ }
118
+
119
+ def format_string(
120
+ self, concept: Concept, anonymized_literal_candidate: str | None = None
121
+ ):
122
+ canonical = concept.canonical
123
+ uri = concept.uri
124
+ literal = concept.literal # noqa
125
+ kw_literal = canonical
126
+ concept_format_string = concept.attributes.get("formatter", None)
127
+ if self.formatters or concept_format_string:
128
+ nr = "not_set"
129
+ literal_data = self.known_entries.get(kw_literal, None)
130
+ if literal_data is not None:
131
+ nr = literal_data.get("nr", "not_set")
132
+ anonymized_literal = literal_data.get(
133
+ "uri", anonymized_literal_candidate
134
+ )
135
+ else:
136
+ nr = self.counter.get(uri, 1)
137
+ anonymized_literal = anonymized_literal_candidate
138
+
139
+ if anonymized_literal is None:
140
+ anonymized_literal = f"{uri}_{nr}"
141
+ anonymized = anonymized_literal # noqa used in f-string
142
+
143
+ if uri not in self.formatters and "default" in self.formatters:
144
+ mapping_kw = "default"
145
+ else:
146
+ mapping_kw = uri
147
+
148
+ if mapping_kw in self.formatters:
149
+ if "fake" in self.formatters[mapping_kw]:
150
+ fake = self.random_instance(concept) # noqa
151
+
152
+ anonymized_literal = eval('f"""' + self.formatters[mapping_kw] + '"""')
153
+ return anonymized_literal
154
+ elif concept_format_string:
155
+ anonymized_literal = eval('f"""' + anonymized_literal + '"""')
156
+ return anonymized_literal
157
+ else:
158
+ return anonymized_literal
159
+
160
+ def update_uri_counter(self, uri: str):
161
+ """update the counter for the given uri"""
162
+ if uri not in self.counter:
163
+ self.counter[uri] = 1
164
+ else:
165
+ self.counter[uri] += 1
166
+
167
+ def anonymize(self, concept: Concept):
168
+ """
169
+ return a formated str or anonymized literal for the given literal with the given uri.
170
+
171
+ :param uri: uri for the given literal
172
+ :type uri: str
173
+ :param literal: literal that needs to be anonymized
174
+ :type literal: str
175
+
176
+ """
177
+ uri = concept.uri
178
+ kw_literal = concept.canonical
179
+ if kw_literal not in self.known_entries:
180
+ if (
181
+ self.pseudonyms
182
+ and uri in self.pseudonyms
183
+ and len(self.pseudonyms[uri]) > 0
184
+ ):
185
+ candidates = self.pseudonyms[uri]
186
+ if candidates:
187
+ self.update_uri_counter(uri)
188
+
189
+ item_index = randint(0, len(candidates) - 1)
190
+ retval = self.format_string(
191
+ concept, anonymized_literal_candidate=candidates[item_index]
192
+ )
193
+ anonymized_literal_ = retval if retval else candidates[item_index]
194
+
195
+ self.known_entries[kw_literal] = {
196
+ "uri": anonymized_literal_,
197
+ "nr": self.counter[uri],
198
+ }
199
+
200
+ # remove this one from the list to avoid duplicates
201
+ del candidates[item_index]
202
+ else:
203
+ self._add_literal_data(concept, kw_literal)
204
+ else:
205
+ self._add_literal_data(concept, kw_literal)
206
+ anonymized_literal = self.known_entries[kw_literal]["uri"]
207
+ return anonymized_literal
208
+
209
+ def write(self, bytes_data: str):
210
+ """write the given data to the output result"""
211
+ self.out += bytes_data
212
+
213
+ def write_location(self, bytes_data: str, concept: Concept):
214
+ """write the given data to the output result"""
215
+ begin_offset = len(self.out)
216
+ self.out += bytes_data
217
+ end_offset = len(self.out)
218
+ self.locations.append(
219
+ {
220
+ KW_BEGIN_OFFSET: begin_offset,
221
+ KW_END_OFFSET: end_offset,
222
+ KW_TEXT: concept.canonical,
223
+ KW_URI: concept.uri,
224
+ KW_ANONYMIZED: bytes_data,
225
+ KW_BYTE_BEGIN_OFFSET: concept.begin_offset,
226
+ KW_BYTE_END_OFFSET: concept.end_offset,
227
+ }
228
+ )
229
+
230
+ def write_substr(self, start: int, end: int | None = None):
231
+ """write the given data to the output result"""
232
+ assert self.input_data_ is not None, "input data is not set"
233
+ if end is None:
234
+ self.out += self.input_data_[start:]
235
+ else:
236
+ self.out += self.input_data_[start:end]
237
+
238
+ def get_value(self):
239
+ """return output result"""
240
+ return {"text": self.out, "locations": self.locations}
241
+
242
+ def get_bytes(self):
243
+ """return output result"""
244
+ return self.out.encode()
@@ -0,0 +1,10 @@
1
+ KW_URI = "uri"
2
+ KW_ANONYMIZED = "anonymized"
3
+ KW_TEXT = "text"
4
+ KW_BEGIN_OFFSET = "begin_offset"
5
+ KW_END_OFFSET = "end_offset"
6
+ KW_BYTE_BEGIN_OFFSET = "byte_begin_offset"
7
+ KW_BYTE_END_OFFSET = "byte_end_offset"
8
+ KW_ANONYMIZED_LITERAL = "anonymized_literal"
9
+ KW_LITERAL = "literal"
10
+ KW_CANONICAL = "canonical"
@@ -0,0 +1,27 @@
1
+ from typing import Protocol, runtime_checkable, Any
2
+ from wowool.annotation import Concept
3
+
4
+
5
+ @runtime_checkable
6
+ class WriterInterface(Protocol):
7
+
8
+ def anonymize(self, concept: Concept):
9
+ """
10
+ return a formatted str or anonymized literal for the given literal with the given uri.
11
+
12
+ :param concept: concept that needs to be anonymized
13
+ :type concept: Concept
14
+ """
15
+ return "***"
16
+
17
+ def write(self, bytes_data: str):
18
+ """write the given data to the output result"""
19
+ pass
20
+
21
+ def clear(self):
22
+ """clear the output data of the writer"""
23
+ pass
24
+
25
+ def get_value(self) -> dict[str, Any]:
26
+ """return the output data of the writer"""
27
+ pass
@@ -0,0 +1,44 @@
1
+ from wowool.document import DocumentInterface
2
+ from wowool.sdk import Pipeline
3
+ from wowool.tools.anonymizer.writer_config import WriterConfig
4
+
5
+
6
+ class Writer:
7
+ """
8
+ Base class for all writers.
9
+ """
10
+
11
+ def __init__(self, writer_config: WriterConfig):
12
+ self.writer_config = writer_config
13
+ self.anonymizers = {}
14
+
15
+ def get_anonymizer(self, mime_type):
16
+ """
17
+ Get the anonymizers for the writer.
18
+ """
19
+ if mime_type not in self.anonymizers:
20
+ if mime_type == "text/plain":
21
+ from wowool.anonymizer.writers.text.writer import Writer as TextWriter
22
+
23
+ self.anonymizers[mime_type] = TextWriter(self.writer_config)
24
+ return self.anonymizers[mime_type]
25
+ elif mime_type == "application/pdf":
26
+ from wowool.anonymizer.writers.pdf.writer import Writer as PdfWriter
27
+
28
+ self.anonymizers[mime_type] = PdfWriter(self.writer_config)
29
+ return self.anonymizers[mime_type]
30
+ else:
31
+ return self.anonymizers[mime_type]
32
+
33
+ def __call__(self, document: DocumentInterface, pipeline: Pipeline):
34
+ """
35
+ Write the document to the specified path.
36
+ """
37
+ if not isinstance(document, DocumentInterface):
38
+ raise TypeError("document must be an instance of DocumentInterface")
39
+ if not isinstance(pipeline, Pipeline):
40
+ raise TypeError("pipeline must be an instance of Pipeline")
41
+ if anonymizer_ := self.get_anonymizer(document.mime_type):
42
+ return anonymizer_(document, pipeline)
43
+ else:
44
+ raise NotImplementedError(f"Writer unknown data type: {document.mime_type}")