wowool-anonymizer 2.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wowool/anonymizer/__init__.py +1 -0
- wowool/anonymizer/core/anonymizer.py +177 -0
- wowool/anonymizer/core/anonymizer_config.py +227 -0
- wowool/anonymizer/core/app_id.py +1 -0
- wowool/anonymizer/core/byte_writer.py +98 -0
- wowool/anonymizer/core/default_writer.py +244 -0
- wowool/anonymizer/core/defines.py +10 -0
- wowool/anonymizer/core/writer_interface.py +27 -0
- wowool/anonymizer/writers/factory/writer.py +46 -0
- wowool/anonymizer/writers/pdf/writer.py +655 -0
- wowool/anonymizer/writers/text/writer.py +39 -0
- wowool/apps/anonymizer/__init__.py +2 -0
- wowool/apps/anonymizer/config.py +4 -0
- wowool/tools/anonymizer/__main__.py +133 -0
- wowool/tools/anonymizer/writer_config.py +10 -0
- wowool/tools/pdf_anonymizer/__main__.py +45 -0
- wowool/tools/pdf_anonymizer/pdf_wow_anonymizer.json +22 -0
- wowool_anonymizer-2.1.1.dist-info/METADATA +340 -0
- wowool_anonymizer-2.1.1.dist-info/RECORD +22 -0
- wowool_anonymizer-2.1.1.dist-info/WHEEL +5 -0
- wowool_anonymizer-2.1.1.dist-info/entry_points.txt +2 -0
- wowool_anonymizer-2.1.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from wowool.anonymizer.core.anonymizer import Anonymizer, DefaultWriter # noqa: F401
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
from wowool.diagnostic import Diagnostics, Diagnostic, DiagnosticType
|
|
2
|
+
from wowool.native.core.engine import Engine, default_engine
|
|
3
|
+
from wowool.native.core import Pipeline
|
|
4
|
+
from wowool.native.core.pipeline_resolver import resolve
|
|
5
|
+
from wowool.annotation import Concept
|
|
6
|
+
from wowool.error import Error
|
|
7
|
+
from random import randint
|
|
8
|
+
from wowool.document import Document
|
|
9
|
+
import copy
|
|
10
|
+
from wowool.utility.apps.decorators import (
|
|
11
|
+
exceptions_to_diagnostics,
|
|
12
|
+
requires_analysis,
|
|
13
|
+
check_requires_concepts,
|
|
14
|
+
)
|
|
15
|
+
from wowool.document.analysis.document import AnalysisDocument
|
|
16
|
+
from wowool.document.factory import Factory
|
|
17
|
+
from wowool.anonymizer.core.app_id import APP_ID
|
|
18
|
+
from logging import getLogger
|
|
19
|
+
import re
|
|
20
|
+
from wowool.anonymizer.core.default_writer import DefaultWriter
|
|
21
|
+
from wowool.document.analysis.utilities import get_pipeline_concepts
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
logger = getLogger(__name__)
|
|
25
|
+
|
|
26
|
+
INVALID_FORMAT_STRING = re.compile("import|from")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def validate_format_string(format_string):
|
|
30
|
+
"""Validate the format string"""
|
|
31
|
+
return True
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def get_default_pseudonyms():
|
|
35
|
+
from wowool.anonymizer.core.anonymizer_config import DEFAULT_PSEUDONYMS
|
|
36
|
+
|
|
37
|
+
return DEFAULT_PSEUDONYMS
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
class Anonymizer:
|
|
41
|
+
"""
|
|
42
|
+
This object traverses the given document and calls back the writer.
|
|
43
|
+
The writer can format the document as you see fit. See DefaultWriter for more information on how to overwrite the Writer
|
|
44
|
+
|
|
45
|
+
:param language: language to process
|
|
46
|
+
:type language: str
|
|
47
|
+
:param domains: list of domains to use.
|
|
48
|
+
:type domains: str
|
|
49
|
+
:param writer: Object used to format your output results.
|
|
50
|
+
:type writer: callable object
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
Using fake names.
|
|
54
|
+
|
|
55
|
+
.. literalinclude:: ../../comp-wowool-anonymizer-py/samples/english_anonymize.py
|
|
56
|
+
:caption: english_anonymize.py
|
|
57
|
+
|
|
58
|
+
Using a formatter, note that you can set a formatter for every entity separately.
|
|
59
|
+
|
|
60
|
+
.. literalinclude:: ../../comp-wowool-anonymizer-py/samples/english_anonymize_formatter.py
|
|
61
|
+
:caption: english_anonymize_formatter.py
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
docs = """
|
|
67
|
+
# Anonymizer
|
|
68
|
+
|
|
69
|
+
The Anonymizer is a tool that can be used to anonymize text data. It can be used to replace sensitive information with fake names or other data. The Anonymizer can be used to replace names, addresses, phone numbers, and other sensitive information with fake data. The Anonymizer can be used to protect the privacy of individuals and organizations by replacing sensitive information with fake data.
|
|
70
|
+
|
|
71
|
+
## Arguments
|
|
72
|
+
|
|
73
|
+
- `annotations`: A list of annotations to anonymize. If not provided, all annotations will be anonymized.
|
|
74
|
+
- `pseudonyms`: A dictionary of fake names to use for each annotation. If not provided, a default list of fake names will be used.
|
|
75
|
+
- `formatters`: A dictionary of formatters to use for each annotation. If not provided, the default formatter will be used.
|
|
76
|
+
|
|
77
|
+
## Example
|
|
78
|
+
|
|
79
|
+
```
|
|
80
|
+
{
|
|
81
|
+
"annotations": ["Person"],
|
|
82
|
+
"pseudonyms": {
|
|
83
|
+
"Person": ["John Doe", "Jane Smith"]
|
|
84
|
+
},
|
|
85
|
+
"formatters": {
|
|
86
|
+
"Person": "###{anonymized_literal}###"
|
|
87
|
+
},
|
|
88
|
+
}
|
|
89
|
+
```
|
|
90
|
+
"""
|
|
91
|
+
|
|
92
|
+
ID = APP_ID
|
|
93
|
+
|
|
94
|
+
def __init__(
|
|
95
|
+
self,
|
|
96
|
+
annotations: list[str] | None = None,
|
|
97
|
+
writer=None,
|
|
98
|
+
pseudonyms: dict[str, list[str]] | None = None,
|
|
99
|
+
formatters: dict[str, str] | None = None,
|
|
100
|
+
fake_locale: list[str] | None = None,
|
|
101
|
+
):
|
|
102
|
+
self.pseudonyms = (
|
|
103
|
+
pseudonyms if pseudonyms is not None else get_default_pseudonyms()
|
|
104
|
+
)
|
|
105
|
+
self.writer = (
|
|
106
|
+
writer
|
|
107
|
+
if writer
|
|
108
|
+
else DefaultWriter(
|
|
109
|
+
pseudonyms=self.pseudonyms,
|
|
110
|
+
formatters=formatters,
|
|
111
|
+
fake_locale=fake_locale,
|
|
112
|
+
)
|
|
113
|
+
)
|
|
114
|
+
self.DEFAULT_SKIP_LIST = set(["Sentence", "VP", "NP", "Event"])
|
|
115
|
+
# We use this so the we do not need all method.
|
|
116
|
+
|
|
117
|
+
if annotations:
|
|
118
|
+
if isinstance(annotations, str):
|
|
119
|
+
self.annotations = set(annotations.split(","))
|
|
120
|
+
self.document_level_annotations = self.annotations
|
|
121
|
+
elif isinstance(annotations, list) or isinstance(annotations, set):
|
|
122
|
+
self.annotations = set(annotations)
|
|
123
|
+
self.document_level_annotations = self.annotations
|
|
124
|
+
else:
|
|
125
|
+
assert False, "Invalid object type for 'annotations' collection"
|
|
126
|
+
else:
|
|
127
|
+
self.annotations = None
|
|
128
|
+
self.document_level_annotations = None
|
|
129
|
+
|
|
130
|
+
@exceptions_to_diagnostics
|
|
131
|
+
@requires_analysis
|
|
132
|
+
def __call__(
|
|
133
|
+
self, document: AnalysisDocument, diagnostics: Diagnostics
|
|
134
|
+
) -> AnalysisDocument:
|
|
135
|
+
"""
|
|
136
|
+
process a InputProvider of str. The result will be stored by the writer.
|
|
137
|
+
|
|
138
|
+
:param input_provider: InputProvider
|
|
139
|
+
:type input_provider: str or InputProvider
|
|
140
|
+
"""
|
|
141
|
+
|
|
142
|
+
if self.document_level_annotations is None:
|
|
143
|
+
self.document_level_annotations = get_pipeline_concepts(document)
|
|
144
|
+
|
|
145
|
+
self.writer.clear()
|
|
146
|
+
self.writer.reset_document_data()
|
|
147
|
+
if document.text is None:
|
|
148
|
+
diagnostics.add(
|
|
149
|
+
Diagnostic(
|
|
150
|
+
document.id,
|
|
151
|
+
"Cannot use the Anonymizer app on a re-entrance document, as it requires the original text.",
|
|
152
|
+
DiagnosticType.Error,
|
|
153
|
+
)
|
|
154
|
+
)
|
|
155
|
+
document.add_diagnostics(self.ID, diagnostics)
|
|
156
|
+
return document
|
|
157
|
+
|
|
158
|
+
self.writer.set_input_data(document.text)
|
|
159
|
+
begin_section_offset = 0
|
|
160
|
+
begin_offset = 0
|
|
161
|
+
for concept in Concept.iter(document):
|
|
162
|
+
if (
|
|
163
|
+
concept.uri in self.document_level_annotations
|
|
164
|
+
and concept.begin_offset >= begin_offset
|
|
165
|
+
):
|
|
166
|
+
anonymized_data = self.writer.anonymize(concept)
|
|
167
|
+
begin_offset = concept.begin_offset
|
|
168
|
+
self.writer.write_substr(begin_section_offset, begin_offset)
|
|
169
|
+
self.writer.write_location(anonymized_data, concept)
|
|
170
|
+
|
|
171
|
+
begin_section_offset = concept.end_offset
|
|
172
|
+
begin_offset = concept.end_offset
|
|
173
|
+
|
|
174
|
+
self.writer.write_substr(begin_section_offset, None)
|
|
175
|
+
if results := self.writer.get_value():
|
|
176
|
+
document.add_results(APP_ID, results)
|
|
177
|
+
return document
|
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
# anonymizer_config.py
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
DEFAULT_PSEUDONYMS = {
|
|
5
|
+
"Person": [
|
|
6
|
+
"Frodo Baggins",
|
|
7
|
+
"Harry Potter",
|
|
8
|
+
"Jack Sparrow",
|
|
9
|
+
"Dexter Morgan",
|
|
10
|
+
"Honey Bonney",
|
|
11
|
+
"Benjamin Button",
|
|
12
|
+
"Marge Simpson",
|
|
13
|
+
"Pepper Potts",
|
|
14
|
+
"Holly Golightly",
|
|
15
|
+
"Don Draper",
|
|
16
|
+
"Forrest Gump",
|
|
17
|
+
"Edna Krabappel",
|
|
18
|
+
"James Bond",
|
|
19
|
+
"Peregrin Took",
|
|
20
|
+
"Sarah Connors",
|
|
21
|
+
"Clarisse Cavendish",
|
|
22
|
+
"Louella Warner",
|
|
23
|
+
"Evelyn Cleveland",
|
|
24
|
+
"Raphaela Donovan",
|
|
25
|
+
"Delta Lombardi",
|
|
26
|
+
"Sadie Mack",
|
|
27
|
+
"Giulia Lawton",
|
|
28
|
+
"Gracie Kilpatrick",
|
|
29
|
+
"Noa Fontaine",
|
|
30
|
+
"Mylah Barney",
|
|
31
|
+
"Zinnia Christensen",
|
|
32
|
+
"Andrea Oakes",
|
|
33
|
+
"Charity Bouvier",
|
|
34
|
+
"Tessa Grenville",
|
|
35
|
+
"Mia Kendrick",
|
|
36
|
+
"Vera Jacobsen",
|
|
37
|
+
"Xara Willoughby",
|
|
38
|
+
"Trixie Cassidy",
|
|
39
|
+
"Evie Ashford",
|
|
40
|
+
"Tia Lawson",
|
|
41
|
+
"Conor Faust",
|
|
42
|
+
"Oliver Senna",
|
|
43
|
+
"Angel Gray",
|
|
44
|
+
"Richard Croix",
|
|
45
|
+
"Conor Silva",
|
|
46
|
+
"Cyprian Baptiste",
|
|
47
|
+
"Lenny Gabriel",
|
|
48
|
+
"Truett Oakes",
|
|
49
|
+
"Justin Lovell",
|
|
50
|
+
"Natha n Watson",
|
|
51
|
+
"Sherwin Faulkner",
|
|
52
|
+
"Osiris Twain",
|
|
53
|
+
"Harper Campbell",
|
|
54
|
+
"Laurence Koch",
|
|
55
|
+
"Diarmaid Garrick",
|
|
56
|
+
"Alvin Bexley",
|
|
57
|
+
"Santos Marlow",
|
|
58
|
+
"Hunt Merrick",
|
|
59
|
+
"Scott Malone",
|
|
60
|
+
"Brad Willis",
|
|
61
|
+
"Margaux Leblanc",
|
|
62
|
+
"Jules Renault",
|
|
63
|
+
"Amandine Gauthier",
|
|
64
|
+
"Théo Lambert",
|
|
65
|
+
"Léa Perrin",
|
|
66
|
+
"Hugo Durand",
|
|
67
|
+
"Capucine Leroy",
|
|
68
|
+
"Romain Mercier",
|
|
69
|
+
"Océane Faure",
|
|
70
|
+
"Valentin Lemoine",
|
|
71
|
+
"Maëlys Dubois",
|
|
72
|
+
"Bastien Roussel",
|
|
73
|
+
"Clémence Caron",
|
|
74
|
+
"Maxime Giraud",
|
|
75
|
+
"Noémie Bertrand",
|
|
76
|
+
"Quentin Lefevre",
|
|
77
|
+
"Inès Martin",
|
|
78
|
+
"Lucas Morel",
|
|
79
|
+
"Chloé Bernard",
|
|
80
|
+
"Adrien Simon",
|
|
81
|
+
],
|
|
82
|
+
"Company": [
|
|
83
|
+
"Acme Corporation",
|
|
84
|
+
"Die Hard Corp",
|
|
85
|
+
"Terminator Ltd",
|
|
86
|
+
"Globex Corporation",
|
|
87
|
+
"Soylent Corp",
|
|
88
|
+
"Duff Beer",
|
|
89
|
+
"Initech",
|
|
90
|
+
"Stark Industries",
|
|
91
|
+
"Ollivander's Wand Shop",
|
|
92
|
+
"Predator Technologies",
|
|
93
|
+
"Wayne Enterprises",
|
|
94
|
+
"Cyberdyne Systems",
|
|
95
|
+
"The New York Inquirer",
|
|
96
|
+
"Umbrella Corporation",
|
|
97
|
+
"Hooli",
|
|
98
|
+
"Bubba Gump",
|
|
99
|
+
"Vehement Capital Partners",
|
|
100
|
+
"Massive Dynamic",
|
|
101
|
+
"Oceanic Airlines",
|
|
102
|
+
"Big Kahuna Burger",
|
|
103
|
+
"Wonka Industries",
|
|
104
|
+
"Mooby’s",
|
|
105
|
+
"Heisler Beer",
|
|
106
|
+
"Krusty Krab",
|
|
107
|
+
"Sterling Cooper",
|
|
108
|
+
"Good Burger",
|
|
109
|
+
"Life Extension",
|
|
110
|
+
"Brawndo",
|
|
111
|
+
"Parallax Corporation",
|
|
112
|
+
"Megadodo Publications",
|
|
113
|
+
"Multi National United",
|
|
114
|
+
"Crimson Permanent Assurance",
|
|
115
|
+
"Spectacular Optical",
|
|
116
|
+
"Adventureland",
|
|
117
|
+
"Lunar Industries",
|
|
118
|
+
"Career Transitions Corporation",
|
|
119
|
+
"Buy N Large",
|
|
120
|
+
"McDowell's",
|
|
121
|
+
"Kiki's Delivery Service",
|
|
122
|
+
"Oscorp",
|
|
123
|
+
"Rekall",
|
|
124
|
+
"Quick Stop",
|
|
125
|
+
"Nakatomi Trading Co.",
|
|
126
|
+
"Bushwood Country Club",
|
|
127
|
+
"Prestige Worldwide",
|
|
128
|
+
"Rare Exports, Inc.",
|
|
129
|
+
"Monsters, Inc.",
|
|
130
|
+
"Amusing Ventures",
|
|
131
|
+
"Hilarious Inc.",
|
|
132
|
+
"Laughable Enterprises",
|
|
133
|
+
"Witty Solutions",
|
|
134
|
+
"Jokester Corporation",
|
|
135
|
+
"Humorous Enterprises",
|
|
136
|
+
"Chuckle Co.",
|
|
137
|
+
"Comical Innovations",
|
|
138
|
+
"Giggles and Grins Inc.",
|
|
139
|
+
"Silly Solutions",
|
|
140
|
+
"Spacely Sprockets",
|
|
141
|
+
"Tyrell Corporation",
|
|
142
|
+
"Gringotts",
|
|
143
|
+
"Wonka Industries",
|
|
144
|
+
"Cyberdyne Systems",
|
|
145
|
+
],
|
|
146
|
+
"Position": [
|
|
147
|
+
"Shredded Cheese Authority",
|
|
148
|
+
"Chief Visionary Officer",
|
|
149
|
+
"Chief People Herder",
|
|
150
|
+
"Wizard",
|
|
151
|
+
"Chief Guru",
|
|
152
|
+
"Ninja Thinker",
|
|
153
|
+
"Road Warrior",
|
|
154
|
+
"Digital Overlord",
|
|
155
|
+
"Cat Behaviour Consultant",
|
|
156
|
+
"Smarties Expert",
|
|
157
|
+
"Space Lawyer",
|
|
158
|
+
"Dark Web Coach",
|
|
159
|
+
"Science Viking",
|
|
160
|
+
"Penguinologist",
|
|
161
|
+
"Rasputin Impersonator",
|
|
162
|
+
"Minister of Silly Walks",
|
|
163
|
+
],
|
|
164
|
+
"Country": [
|
|
165
|
+
"Borduria",
|
|
166
|
+
"Syldavia",
|
|
167
|
+
"San Teodoros",
|
|
168
|
+
"São Rico",
|
|
169
|
+
"Arendelle",
|
|
170
|
+
"Berzerkistan",
|
|
171
|
+
"Wakanda",
|
|
172
|
+
"Zamunda",
|
|
173
|
+
"Freedonia",
|
|
174
|
+
"Syldavia",
|
|
175
|
+
"Sokovia",
|
|
176
|
+
"Agrabah",
|
|
177
|
+
"Ruritania",
|
|
178
|
+
"Pianosa",
|
|
179
|
+
"Borduria",
|
|
180
|
+
"Nambutu",
|
|
181
|
+
"Zubrowka",
|
|
182
|
+
"Kumandra",
|
|
183
|
+
],
|
|
184
|
+
"Organization": [
|
|
185
|
+
"Combined Miniature Deterrent Force",
|
|
186
|
+
"Galactic Alliance Security",
|
|
187
|
+
"Brotherhood of the Ring",
|
|
188
|
+
"Ministry of Magic",
|
|
189
|
+
"Judean Liberation Front",
|
|
190
|
+
"People's Revolutionary Front",
|
|
191
|
+
"Highest Anti Terrorism Effort",
|
|
192
|
+
"Democratic Order of Planets",
|
|
193
|
+
"Global Liberation Army",
|
|
194
|
+
"Office of Secret Intelligence",
|
|
195
|
+
],
|
|
196
|
+
"City": [
|
|
197
|
+
"National City",
|
|
198
|
+
"Gotham City",
|
|
199
|
+
"Sunnydale",
|
|
200
|
+
"Bludhaven",
|
|
201
|
+
"Basin City",
|
|
202
|
+
"Riverdale",
|
|
203
|
+
"Smallville",
|
|
204
|
+
"Metropolis",
|
|
205
|
+
"Springfield",
|
|
206
|
+
"Castle Rock",
|
|
207
|
+
"Emerald City",
|
|
208
|
+
"Mayberry",
|
|
209
|
+
"Seabrook",
|
|
210
|
+
"Aberdale",
|
|
211
|
+
"South Park",
|
|
212
|
+
"Shangri-La",
|
|
213
|
+
"Middlemarch",
|
|
214
|
+
"Lankhmar",
|
|
215
|
+
"Los Santos",
|
|
216
|
+
"Liberty City",
|
|
217
|
+
"Avalon",
|
|
218
|
+
"Walmington-on-Sea",
|
|
219
|
+
"Metropolis",
|
|
220
|
+
"Raccoon City",
|
|
221
|
+
"Duckburg",
|
|
222
|
+
"Woodbury",
|
|
223
|
+
"Atlantis",
|
|
224
|
+
],
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
DEFAULT_ANNOTATIONS = "Person,Company,Position,Country,City"
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
APP_ID = "wowool_anonymizer"
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
from wowool.anonymizer.core.writer_interface import WriterInterface
|
|
2
|
+
from random import randint
|
|
3
|
+
import copy
|
|
4
|
+
from wowool.annotation import Concept
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ByteWriter(WriterInterface):
|
|
8
|
+
"""
|
|
9
|
+
This object will rewrite the original document.
|
|
10
|
+
|
|
11
|
+
:param dummies: A dictionary that has a list of fake names for a given uri. Example: { "Person" : ["Hulk","Flash"] }
|
|
12
|
+
:type dummies: dict( uri: list(names) )
|
|
13
|
+
:param formatters: Format how an entity should be displayed in the document. Example: { "Person" : "{anonymized_literal}" }
|
|
14
|
+
:type formatters: dict(uri, fstring format), internal variables that can be used. anonymized_literal, uri, nr, literal
|
|
15
|
+
"""
|
|
16
|
+
|
|
17
|
+
def __init__(
|
|
18
|
+
self,
|
|
19
|
+
dummies={},
|
|
20
|
+
formatters=None,
|
|
21
|
+
using_byte_offset=False,
|
|
22
|
+
):
|
|
23
|
+
self.dummies = copy.deepcopy(dummies)
|
|
24
|
+
self.formatters = formatters
|
|
25
|
+
self.clear()
|
|
26
|
+
self.reset_document_data()
|
|
27
|
+
self.using_byte_offset = using_byte_offset
|
|
28
|
+
|
|
29
|
+
def reset_document_data(self):
|
|
30
|
+
"""clear the known_entries of the writer, which means that previous entries will not be the same in the following document"""
|
|
31
|
+
self.counter, self.known_entries = {}, {}
|
|
32
|
+
|
|
33
|
+
def clear(self):
|
|
34
|
+
"""clear the output data of the writer"""
|
|
35
|
+
if self.using_byte_offset:
|
|
36
|
+
self.out = b""
|
|
37
|
+
else:
|
|
38
|
+
self.out = ""
|
|
39
|
+
|
|
40
|
+
def _add_literal_data(self, uri, literal):
|
|
41
|
+
if uri not in self.counter:
|
|
42
|
+
self.counter[uri] = 0
|
|
43
|
+
self.counter[uri] += 1
|
|
44
|
+
self.known_entries[literal] = {
|
|
45
|
+
"uri": f"{uri}_{self.counter[uri]}",
|
|
46
|
+
"nr": self.counter[uri],
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
def anonymize(self, concept: Concept):
|
|
50
|
+
"""
|
|
51
|
+
return a formated str or anonymized literal for the given literal with the given uri.
|
|
52
|
+
|
|
53
|
+
:param uri: uri for the given literal
|
|
54
|
+
:type uri: str
|
|
55
|
+
:param literal: literal that needs to be anonymized
|
|
56
|
+
:type literal: str
|
|
57
|
+
|
|
58
|
+
"""
|
|
59
|
+
uri, literal = concept.uri, concept.literal
|
|
60
|
+
if literal not in self.known_entries:
|
|
61
|
+
if uri in self.dummies and len(self.dummies[uri]) > 0:
|
|
62
|
+
candidates = self.dummies[uri]
|
|
63
|
+
if candidates:
|
|
64
|
+
|
|
65
|
+
item_index = randint(0, len(candidates) - 1)
|
|
66
|
+
self.known_entries[literal] = {
|
|
67
|
+
"uri": candidates[item_index],
|
|
68
|
+
"nr": "0",
|
|
69
|
+
}
|
|
70
|
+
# remove this one from the list to avoid duplicates
|
|
71
|
+
del candidates[item_index]
|
|
72
|
+
else:
|
|
73
|
+
self._add_literal_data(uri, literal)
|
|
74
|
+
else:
|
|
75
|
+
self._add_literal_data(uri, literal)
|
|
76
|
+
anonymized_literal = self.known_entries[literal]["uri"]
|
|
77
|
+
|
|
78
|
+
if self.formatters:
|
|
79
|
+
nr = "not_set"
|
|
80
|
+
if "nr" in self.known_entries[literal]:
|
|
81
|
+
nr = self.known_entries[literal]["nr"]
|
|
82
|
+
if uri in self.formatters:
|
|
83
|
+
return eval('f"' + self.formatters[uri] + '"')
|
|
84
|
+
elif "all" in self.formatters:
|
|
85
|
+
return eval('f"' + self.formatters["all"] + '"')
|
|
86
|
+
return anonymized_literal
|
|
87
|
+
|
|
88
|
+
def write(self, bytes_data):
|
|
89
|
+
"""write the given data to the output result"""
|
|
90
|
+
self.out += bytes_data
|
|
91
|
+
|
|
92
|
+
def get_value(self):
|
|
93
|
+
"""return output result"""
|
|
94
|
+
return self.out.decode()
|
|
95
|
+
|
|
96
|
+
def get_bytes(self):
|
|
97
|
+
"""return output result"""
|
|
98
|
+
return self.out
|