wowool-anonymizer 2.1.1.dev13__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wowool/anonymizer/__init__.py +1 -0
- wowool/anonymizer/core/anonymizer.py +164 -0
- wowool/anonymizer/core/anonymizer_config.py +227 -0
- wowool/anonymizer/core/app_id.py +1 -0
- wowool/anonymizer/core/byte_writer.py +98 -0
- wowool/anonymizer/core/default_writer.py +244 -0
- wowool/anonymizer/core/defines.py +10 -0
- wowool/anonymizer/core/writer_interface.py +27 -0
- wowool/anonymizer/writers/factory/writer.py +44 -0
- wowool/anonymizer/writers/pdf/writer.py +605 -0
- wowool/anonymizer/writers/text/writer.py +39 -0
- wowool/apps/anonymizer/__init__.py +2 -0
- wowool/apps/anonymizer/config.py +4 -0
- wowool/tools/anonymizer/__main__.py +123 -0
- wowool/tools/anonymizer/writer_config.py +10 -0
- wowool/tools/pdf_anonymizer/__main__.py +45 -0
- wowool/tools/pdf_anonymizer/pdf_wow_anonymizer.json +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/METADATA +19 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/RECORD +22 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/WHEEL +5 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/entry_points.txt +2 -0
- wowool_anonymizer-2.1.1.dev13.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,605 @@
|
|
|
1
|
+
#!python
|
|
2
|
+
from dataclasses import dataclass, field
|
|
3
|
+
from wowool.sdk import Pipeline
|
|
4
|
+
from reportlab.pdfgen import canvas
|
|
5
|
+
from reportlab.lib.pagesizes import letter, A4
|
|
6
|
+
from PIL import Image # optional, for getting image dimensions
|
|
7
|
+
|
|
8
|
+
from reportlab.pdfbase import pdfmetrics
|
|
9
|
+
from reportlab.pdfbase.ttfonts import TTFont
|
|
10
|
+
from fontTools.ttLib import TTCollection
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from wowool.io.pdf.pdfcore import extract_results, ParseOptions
|
|
13
|
+
from wowool.io.pdf.objects import (
|
|
14
|
+
TextBlock,
|
|
15
|
+
TextBlockGroup,
|
|
16
|
+
RectGroup,
|
|
17
|
+
ImageBlock,
|
|
18
|
+
FigureGroup,
|
|
19
|
+
VisualLineSeparator,
|
|
20
|
+
)
|
|
21
|
+
from wowool.sdk import Pipeline
|
|
22
|
+
import traceback
|
|
23
|
+
from reportlab.pdfbase import pdfmetrics
|
|
24
|
+
from reportlab.pdfbase.ttfonts import TTFont
|
|
25
|
+
import re
|
|
26
|
+
from collections import OrderedDict
|
|
27
|
+
from pdfminer.layout import (
|
|
28
|
+
LTTextLineVertical,
|
|
29
|
+
)
|
|
30
|
+
from wowool.io.pdf.pdfcore import is_text_bold
|
|
31
|
+
from wowool.document import DocumentInterface
|
|
32
|
+
from wowool.sdk import Pipeline
|
|
33
|
+
from wowool.anonymizer.core.anonymizer import Anonymizer, DefaultWriter
|
|
34
|
+
from sys import stderr
|
|
35
|
+
from pathlib import Path
|
|
36
|
+
from wowool.tools.anonymizer.writer_config import WriterConfig
|
|
37
|
+
from wowool.anonymizer.core.defines import (
|
|
38
|
+
KW_URI,
|
|
39
|
+
KW_ANONYMIZED,
|
|
40
|
+
KW_TEXT,
|
|
41
|
+
KW_BEGIN_OFFSET,
|
|
42
|
+
KW_END_OFFSET,
|
|
43
|
+
KW_BYTE_BEGIN_OFFSET,
|
|
44
|
+
KW_BYTE_END_OFFSET,
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
# c.linkURL("https://www.example.com", (100, 700, 100, 15), relative=1)
|
|
49
|
+
|
|
50
|
+
split_font_name_pattern = re.compile(r"[,\- ]")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
font_rename_map = OrderedDict(
|
|
54
|
+
{
|
|
55
|
+
"timesnewromanpsmt": "timesnewroman",
|
|
56
|
+
"timesnewromanps-boldmt": "timesnewromanbold",
|
|
57
|
+
"timesnewromanpsboldmt": "timesnewromanbold",
|
|
58
|
+
"timesnewromanps-italicmt": "timesnewromanitalic",
|
|
59
|
+
"timesnewromanps-bolditalicmt": "timesnewromanbolditalic",
|
|
60
|
+
"helveticaneue": "helvetica",
|
|
61
|
+
"helveticaneue-bold": "helvetica-bold",
|
|
62
|
+
}
|
|
63
|
+
)
|
|
64
|
+
reverse_font_rename_map = OrderedDict({"ebgaramond": "garamond"})
|
|
65
|
+
|
|
66
|
+
font_locations = [
|
|
67
|
+
"/System/Library/Fonts/Supplemental",
|
|
68
|
+
"/System/Library/Fonts/",
|
|
69
|
+
"~/Library/Fonts/",
|
|
70
|
+
]
|
|
71
|
+
font_locations = [Path(loc).expanduser() for loc in font_locations]
|
|
72
|
+
already_registered_fonts = set(pdfmetrics.getRegisteredFontNames())
|
|
73
|
+
|
|
74
|
+
pdf_font_mapping = OrderedDict()
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def clean_font_name(fontname):
|
|
78
|
+
"""
|
|
79
|
+
Cleans the font name by removing unwanted characters.
|
|
80
|
+
"""
|
|
81
|
+
# Remove unwanted characters and split the font name
|
|
82
|
+
font_parts = split_font_name_pattern.split(fontname)
|
|
83
|
+
# Join the cleaned parts back together
|
|
84
|
+
cleaned_fontname = "".join(font_parts)
|
|
85
|
+
return cleaned_fontname
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
@dataclass
|
|
89
|
+
class FontInfoObject:
|
|
90
|
+
filename: Path
|
|
91
|
+
font_type: str = "ttf"
|
|
92
|
+
ttc_font: TTFont | None = None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def load_fonts_mapping_table(locations: list[Path]):
|
|
96
|
+
font_folder = Path("fonts").absolute()
|
|
97
|
+
font_folder.mkdir(exist_ok=True)
|
|
98
|
+
for folder in locations:
|
|
99
|
+
for fn in Path(folder).glob("**/*.tt[fc]"):
|
|
100
|
+
|
|
101
|
+
if fn.suffix == ".ttc":
|
|
102
|
+
ttc = TTCollection(fn)
|
|
103
|
+
for ttc_font in ttc:
|
|
104
|
+
# Try to get a meaningful name from the font
|
|
105
|
+
try:
|
|
106
|
+
font_name = ttc_font["name"].getDebugName(4)
|
|
107
|
+
except:
|
|
108
|
+
# Fallback to a generic name with index
|
|
109
|
+
continue
|
|
110
|
+
fontname = clean_font_name(font_name.lower())
|
|
111
|
+
# Create output path
|
|
112
|
+
output_path = font_folder / f"{font_name}.ttf"
|
|
113
|
+
pdf_font_mapping[fontname] = FontInfoObject(
|
|
114
|
+
font_type="ttc", filename=output_path, ttc_font=ttc_font
|
|
115
|
+
)
|
|
116
|
+
else:
|
|
117
|
+
fontname = fn.stem.lower()
|
|
118
|
+
|
|
119
|
+
fparts = fontname.split("-")
|
|
120
|
+
if len(fparts) == 2:
|
|
121
|
+
fparts[0] = reverse_font_rename_map.get(fparts[0], fparts[0])
|
|
122
|
+
fontname = "-".join(fparts)
|
|
123
|
+
if fparts[1] == "regular":
|
|
124
|
+
pdf_font_mapping[fparts[0]] = FontInfoObject(filename=fn)
|
|
125
|
+
fontname = reverse_font_rename_map.get(fontname, fontname)
|
|
126
|
+
fontname = clean_font_name(fontname)
|
|
127
|
+
pdf_font_mapping[fontname] = FontInfoObject(filename=fn)
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
load_fonts_mapping_table(font_locations)
|
|
131
|
+
|
|
132
|
+
# for font in pdf_font_mapping.items():
|
|
133
|
+
# print(font[0], font[1].stem)
|
|
134
|
+
|
|
135
|
+
# print("--------------------")
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
# clean_font_name_pattern = re.compile(r"^(*.)ms$")
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def check_image_rotation(block):
|
|
142
|
+
try:
|
|
143
|
+
img = Image.open(block.filename)
|
|
144
|
+
|
|
145
|
+
# Check for EXIF rotation data
|
|
146
|
+
if hasattr(img, "_getexif") and img._getexif():
|
|
147
|
+
exif = dict(img._getexif().items())
|
|
148
|
+
orientation = exif.get(274, 1) # 274 is the orientation tag
|
|
149
|
+
if orientation != 1:
|
|
150
|
+
return True, orientation
|
|
151
|
+
|
|
152
|
+
# Check aspect ratio against block dimensions
|
|
153
|
+
img_ratio = img.width / img.height
|
|
154
|
+
block_ratio = (block.x1 - block.x0) / (block.y0 - block.y1)
|
|
155
|
+
|
|
156
|
+
# If ratios differ significantly, image may be rotated
|
|
157
|
+
if abs(img_ratio - block_ratio) > 0.1:
|
|
158
|
+
return True, "aspect_ratio_mismatch"
|
|
159
|
+
|
|
160
|
+
return False, None
|
|
161
|
+
except Exception as e:
|
|
162
|
+
print(f"Error checking image rotation: {e}")
|
|
163
|
+
return False, None
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def check_and_register_fonts(locations: list[Path], name):
|
|
167
|
+
"""
|
|
168
|
+
Register fonts from a given folder.
|
|
169
|
+
"""
|
|
170
|
+
|
|
171
|
+
# if name.startswith("timesnewroman"):
|
|
172
|
+
# print("Garamond bold")
|
|
173
|
+
|
|
174
|
+
fontname = font_rename_map.get(name, name)
|
|
175
|
+
if fontname in already_registered_fonts:
|
|
176
|
+
return fontname
|
|
177
|
+
|
|
178
|
+
if fontname in pdf_font_mapping:
|
|
179
|
+
font_info = pdf_font_mapping[fontname]
|
|
180
|
+
if font_info.font_type == "ttc":
|
|
181
|
+
# Register the font from the TTC collection
|
|
182
|
+
if not font_info.filename.exists():
|
|
183
|
+
font_info.ttc_font.save(font_info.filename)
|
|
184
|
+
pdfmetrics.registerFont(TTFont(fontname, font_info.filename))
|
|
185
|
+
else:
|
|
186
|
+
pdfmetrics.registerFont(
|
|
187
|
+
TTFont(fontname, pdf_font_mapping[fontname].filename)
|
|
188
|
+
)
|
|
189
|
+
already_registered_fonts.add(fontname)
|
|
190
|
+
|
|
191
|
+
return fontname
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def is_url_link(text):
|
|
195
|
+
if (
|
|
196
|
+
text.startswith("www")
|
|
197
|
+
or text.startswith("http://")
|
|
198
|
+
or text.startswith("https://")
|
|
199
|
+
):
|
|
200
|
+
return True
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class PDFWriter:
|
|
204
|
+
def __init__(self, filename, pagesize):
|
|
205
|
+
self.filename = filename
|
|
206
|
+
self.canvas = canvas.Canvas(filename, pagesize=A4)
|
|
207
|
+
self.pagesize = pagesize
|
|
208
|
+
self.text = ""
|
|
209
|
+
self.font_name = "Helvetica"
|
|
210
|
+
self.font_size = 12
|
|
211
|
+
|
|
212
|
+
def draw_text(self, block):
|
|
213
|
+
fontname_parts = block.fontname.split("+")
|
|
214
|
+
if len(fontname_parts) > 1:
|
|
215
|
+
fontname = fontname_parts[1].lower()
|
|
216
|
+
else:
|
|
217
|
+
fontname = block.fontname.lower()
|
|
218
|
+
# print(f"fontname: {fontname}")
|
|
219
|
+
try:
|
|
220
|
+
fontname = clean_font_name(fontname)
|
|
221
|
+
fontname = check_and_register_fonts(locations=font_locations, name=fontname)
|
|
222
|
+
self.font_size = block.obj.height
|
|
223
|
+
self.canvas.setFont(fontname, block.obj.height)
|
|
224
|
+
self.font_name = fontname
|
|
225
|
+
except Exception as e:
|
|
226
|
+
print(
|
|
227
|
+
f"Warning: could not set font {fontname}: {e}, {self.font_name},{self.font_size} {block.text}"
|
|
228
|
+
)
|
|
229
|
+
self.canvas.setFont(self.font_name, self.font_size)
|
|
230
|
+
# self.canvas.setFont("OpenSans", block.obj.height)
|
|
231
|
+
|
|
232
|
+
if isinstance(block.obj, LTTextLineVertical) and block.vertical:
|
|
233
|
+
# Save the current graphics state
|
|
234
|
+
self.canvas.saveState()
|
|
235
|
+
|
|
236
|
+
# Move to the point where text should be positioned
|
|
237
|
+
self.canvas.translate(block.x0, block.y1)
|
|
238
|
+
|
|
239
|
+
# Rotate the canvas (90 degrees for true vertical)
|
|
240
|
+
self.canvas.rotate(90)
|
|
241
|
+
|
|
242
|
+
# Set font properties
|
|
243
|
+
self.font_size = block.x1 - block.x0
|
|
244
|
+
self.canvas.setFont(self.font_name, self.font_size)
|
|
245
|
+
|
|
246
|
+
# Draw at origin (0,0) since we've already translated to the desired position
|
|
247
|
+
self.canvas.drawString(0, 0, block.text)
|
|
248
|
+
|
|
249
|
+
# Restore to the saved state (undoes rotation and translation)
|
|
250
|
+
self.canvas.restoreState()
|
|
251
|
+
else:
|
|
252
|
+
# if "1. det är" in block.text:
|
|
253
|
+
# self.canvas.setFillColorRGB(1, 0, 0)
|
|
254
|
+
# self.canvas.setStrokeColorRGB(1, 0, 0)
|
|
255
|
+
# else:
|
|
256
|
+
# self.canvas.setFillColorRGB(0, 0, 0)
|
|
257
|
+
# self.canvas.setStrokeColorRGB(0, 0, 0)
|
|
258
|
+
if isinstance(block.obj, LTTextLineVertical):
|
|
259
|
+
# print(f"Vertical text: {block.text}")
|
|
260
|
+
self.canvas.setFont(self.font_name, block.obj._objs[0].height)
|
|
261
|
+
|
|
262
|
+
if is_url_link(block.text):
|
|
263
|
+
self.canvas.saveState()
|
|
264
|
+
self.canvas.setStrokeColorRGB(1, 0, 2)
|
|
265
|
+
self.canvas.drawString(block.x0, block.yl, block.text)
|
|
266
|
+
self.canvas.restoreState()
|
|
267
|
+
else:
|
|
268
|
+
self.canvas.drawString(block.x0, block.yl, block.text)
|
|
269
|
+
|
|
270
|
+
# if is_url_link(block.text):
|
|
271
|
+
# # print(f"WWW: {block.obj._objs[0]}")
|
|
272
|
+
# from reportlab.lib.colors import blue, Color
|
|
273
|
+
|
|
274
|
+
# custom_color = Color(0.8, 0.1, 0.3) # pinkish red
|
|
275
|
+
|
|
276
|
+
# self.canvas.linkURL(
|
|
277
|
+
# block.text,
|
|
278
|
+
# (block.x0, block.y1, block.x1, block.y0),
|
|
279
|
+
# relative=1,
|
|
280
|
+
# color=custom_color,
|
|
281
|
+
# thickness=1,
|
|
282
|
+
# )
|
|
283
|
+
|
|
284
|
+
def draw_blocks(self, blocks):
|
|
285
|
+
for block in blocks:
|
|
286
|
+
if isinstance(block, TextBlock):
|
|
287
|
+
# print(f"Block: {block}")
|
|
288
|
+
self.draw_text(block)
|
|
289
|
+
self.text += block.text
|
|
290
|
+
elif isinstance(block, TextBlockGroup):
|
|
291
|
+
# print(f"Block: {block}")
|
|
292
|
+
self.draw_blocks(block.text_blocks)
|
|
293
|
+
elif isinstance(block, RectGroup):
|
|
294
|
+
# print(f"Block: {block}")
|
|
295
|
+
self.draw_blocks(block.text_blocks)
|
|
296
|
+
elif isinstance(block, FigureGroup):
|
|
297
|
+
self.draw_blocks(block.text_blocks)
|
|
298
|
+
elif isinstance(block, ImageBlock):
|
|
299
|
+
self.draw_image(block)
|
|
300
|
+
elif isinstance(block, VisualLineSeparator):
|
|
301
|
+
else:
|
|
302
|
+
print(f"Unknown block type: {type(block)}")
|
|
303
|
+
|
|
304
|
+
def draw_images(self, blocks):
|
|
305
|
+
for block in blocks:
|
|
306
|
+
if isinstance(block, TextBlock):
|
|
307
|
+
# print(f"Block: {block}")
|
|
308
|
+
# self.draw_text(block)
|
|
309
|
+
# self.text += block.text
|
|
310
|
+
pass
|
|
311
|
+
elif isinstance(block, TextBlockGroup):
|
|
312
|
+
# print(f"Block: {block}")
|
|
313
|
+
self.draw_images(block.text_blocks)
|
|
314
|
+
elif isinstance(block, RectGroup):
|
|
315
|
+
# print(f"Block: {block}")
|
|
316
|
+
self.draw_images(block.text_blocks)
|
|
317
|
+
elif isinstance(block, FigureGroup):
|
|
318
|
+
self.draw_images(block.text_blocks)
|
|
319
|
+
elif isinstance(block, ImageBlock):
|
|
320
|
+
self.draw_image(block)
|
|
321
|
+
else:
|
|
322
|
+
print(f"Unknown block type: {type(block)}")
|
|
323
|
+
|
|
324
|
+
def draw_text_blocks(self, blocks):
|
|
325
|
+
for block in blocks:
|
|
326
|
+
if isinstance(block, TextBlock):
|
|
327
|
+
# print(f"Block: {block}")
|
|
328
|
+
self.draw_text(block)
|
|
329
|
+
self.text += block.text
|
|
330
|
+
elif isinstance(block, TextBlockGroup):
|
|
331
|
+
# print(f"Block: {block}")
|
|
332
|
+
self.draw_text_blocks(block.text_blocks)
|
|
333
|
+
elif isinstance(block, RectGroup):
|
|
334
|
+
# print(f"Block: {block}")
|
|
335
|
+
self.draw_text_blocks(block.text_blocks)
|
|
336
|
+
elif isinstance(block, FigureGroup):
|
|
337
|
+
self.draw_text_blocks(block.text_blocks)
|
|
338
|
+
elif isinstance(block, ImageBlock):
|
|
339
|
+
# self.draw_image(block)
|
|
340
|
+
pass
|
|
341
|
+
else:
|
|
342
|
+
print(f"Unknown block type: {type(block)}")
|
|
343
|
+
|
|
344
|
+
def draw_image(self, block):
|
|
345
|
+
try:
|
|
346
|
+
img = Image.open(block.filename)
|
|
347
|
+
is_rotated, rotation_value = check_image_rotation(block)
|
|
348
|
+
|
|
349
|
+
if is_rotated:
|
|
350
|
+
# Save current state
|
|
351
|
+
self.canvas.saveState()
|
|
352
|
+
|
|
353
|
+
# Calculate dimensions - important to swap width/height for 90/270 rotations
|
|
354
|
+
width = block.x1 - block.x0
|
|
355
|
+
height = block.y0 - block.y1
|
|
356
|
+
|
|
357
|
+
# For 90 or 270 degree rotations, swap dimensions
|
|
358
|
+
if isinstance(rotation_value, int):
|
|
359
|
+
rotation_degrees = {3: 180, 6: 90, 8: 270}.get(rotation_value, 90)
|
|
360
|
+
else:
|
|
361
|
+
rotation_degrees = 90
|
|
362
|
+
|
|
363
|
+
# For 90 or 270 degree rotations, swap width and height
|
|
364
|
+
if rotation_degrees in (90, 270):
|
|
365
|
+
# Need to recalculate position to maintain alignment
|
|
366
|
+
center_x = block.x0 + width / 2
|
|
367
|
+
center_y = block.y1 + height / 2
|
|
368
|
+
|
|
369
|
+
# Properly center the image around its midpoint
|
|
370
|
+
self.canvas.translate(center_x, center_y)
|
|
371
|
+
self.canvas.rotate(rotation_degrees)
|
|
372
|
+
|
|
373
|
+
# Important: Note the negative coordinates to maintain proper positioning
|
|
374
|
+
# and swapped dimensions for proper scaling
|
|
375
|
+
self.canvas.drawImage(
|
|
376
|
+
block.filename,
|
|
377
|
+
-height / 2,
|
|
378
|
+
-width / 2, # Centered coordinates
|
|
379
|
+
width=height, # Swap width/height
|
|
380
|
+
height=width, # Swap width/height
|
|
381
|
+
mask="auto",
|
|
382
|
+
)
|
|
383
|
+
else:
|
|
384
|
+
# Handle 180 degree rotation
|
|
385
|
+
self.canvas.translate(block.x0 + width / 2, block.y1 + height / 2)
|
|
386
|
+
self.canvas.rotate(rotation_degrees)
|
|
387
|
+
self.canvas.drawImage(
|
|
388
|
+
block.filename,
|
|
389
|
+
-width / 2,
|
|
390
|
+
-height / 2, # Center coordinates
|
|
391
|
+
width=width,
|
|
392
|
+
height=height,
|
|
393
|
+
mask="auto",
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
# Restore state
|
|
397
|
+
self.canvas.restoreState()
|
|
398
|
+
else:
|
|
399
|
+
# Unrotated image - original code
|
|
400
|
+
self.canvas.drawImage(
|
|
401
|
+
block.filename,
|
|
402
|
+
block.x0,
|
|
403
|
+
block.y1,
|
|
404
|
+
width=block.x1 - block.x0,
|
|
405
|
+
height=block.y0 - block.y1,
|
|
406
|
+
mask="auto",
|
|
407
|
+
)
|
|
408
|
+
except Exception as e:
|
|
409
|
+
print(f"Error drawing image {block.filename}: {e}")
|
|
410
|
+
|
|
411
|
+
def new_page(self):
|
|
412
|
+
self.canvas.showPage()
|
|
413
|
+
|
|
414
|
+
def save(self):
|
|
415
|
+
self.canvas.save()
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
@dataclass
|
|
419
|
+
class WTextBlock:
|
|
420
|
+
begin_offset: int
|
|
421
|
+
end_offset: int
|
|
422
|
+
block: any
|
|
423
|
+
page: any
|
|
424
|
+
|
|
425
|
+
def __repr__(self):
|
|
426
|
+
return (
|
|
427
|
+
f"WTextBlock({self.begin_offset}, {self.end_offset}, '{self.block.text}' )"
|
|
428
|
+
)
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
@dataclass
|
|
432
|
+
class RwTextBlock:
|
|
433
|
+
rw_block: TextBlock
|
|
434
|
+
locations: list = field(default_factory=list)
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
@dataclass
|
|
438
|
+
class WDocument:
|
|
439
|
+
text: str = ""
|
|
440
|
+
blocks: list = field(default_factory=list)
|
|
441
|
+
prev_bold_status: bool = False
|
|
442
|
+
cur_bold_status: bool = False
|
|
443
|
+
|
|
444
|
+
def add(self, block, page):
|
|
445
|
+
if isinstance(block, TextBlock):
|
|
446
|
+
self.cur_bold_status = is_text_bold(block)
|
|
447
|
+
if not self.cur_bold_status and self.prev_bold_status:
|
|
448
|
+
self.text += "\n"
|
|
449
|
+
|
|
450
|
+
start_offset = len(self.text)
|
|
451
|
+
# Check if the block is empty
|
|
452
|
+
tb = WTextBlock(
|
|
453
|
+
start_offset, start_offset + len(block.text), block=block, page=page
|
|
454
|
+
)
|
|
455
|
+
|
|
456
|
+
# print("WB", block)
|
|
457
|
+
if is_text_bold(block):
|
|
458
|
+
self.prev_bold_status = True
|
|
459
|
+
else:
|
|
460
|
+
self.prev_bold_status = False
|
|
461
|
+
|
|
462
|
+
self.blocks.append(tb)
|
|
463
|
+
|
|
464
|
+
self.text += block.text
|
|
465
|
+
self.text += "\n"
|
|
466
|
+
self.prev_bold_status = self.cur_bold_status
|
|
467
|
+
|
|
468
|
+
# self.input_text_blocks.append(block)
|
|
469
|
+
elif isinstance(block, TextBlockGroup):
|
|
470
|
+
for b in block.text_blocks:
|
|
471
|
+
self.add(b, page)
|
|
472
|
+
self.text += "\n"
|
|
473
|
+
elif isinstance(block, RectGroup):
|
|
474
|
+
for b in block.text_blocks:
|
|
475
|
+
self.add(b, page)
|
|
476
|
+
self.text += "\n"
|
|
477
|
+
elif isinstance(block, FigureGroup):
|
|
478
|
+
for b in block.text_blocks:
|
|
479
|
+
self.add(b, page)
|
|
480
|
+
self.text += "\n"
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def find_next_block(blocks, start_offset, offset):
|
|
484
|
+
"""
|
|
485
|
+
Finds the next block in the list of blocks starting from the given offset.
|
|
486
|
+
"""
|
|
487
|
+
len_blocks = len(blocks)
|
|
488
|
+
for idx in range(start_offset, len_blocks):
|
|
489
|
+
if blocks[idx].begin_offset <= offset < blocks[idx].end_offset:
|
|
490
|
+
return idx, blocks[idx]
|
|
491
|
+
return -1, None
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def anonymize_pdf_document_results(results):
|
|
495
|
+
input_info = WDocument()
|
|
496
|
+
for page in results["document"].pages:
|
|
497
|
+
input_info.text += (
|
|
498
|
+
f"\n-------------------------{page.page_number}-------------------------\n"
|
|
499
|
+
)
|
|
500
|
+
for block in page.layout_blocks:
|
|
501
|
+
input_info.add(block, page)
|
|
502
|
+
return input_info
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
class Writer:
|
|
506
|
+
def __init__(self, writer_config: WriterConfig):
|
|
507
|
+
self.writer_config = writer_config
|
|
508
|
+
self.suffix = "suffix" if writer_config.suffix is None else writer_config.suffix
|
|
509
|
+
self.writer = DefaultWriter(
|
|
510
|
+
writer_config.pseudonyms,
|
|
511
|
+
writer_config.formatters,
|
|
512
|
+
)
|
|
513
|
+
self.anonymizer = Anonymizer(
|
|
514
|
+
writer_config.annotations,
|
|
515
|
+
self.writer,
|
|
516
|
+
)
|
|
517
|
+
|
|
518
|
+
def __call__(self, document: DocumentInterface, pipeline: Pipeline) -> Path:
|
|
519
|
+
pdf_path = Path(document.id)
|
|
520
|
+
output_filename = pdf_path.with_suffix(".anonymized.pdf")
|
|
521
|
+
|
|
522
|
+
writer = PDFWriter(str(output_filename), pagesize=A4)
|
|
523
|
+
# print(f"Creating PDF: {output_filename} {A4}")
|
|
524
|
+
# get the parse tree
|
|
525
|
+
parse_options = ParseOptions(remove_footer_header=False)
|
|
526
|
+
document_tree = extract_results(pdf_path, options=parse_options)
|
|
527
|
+
|
|
528
|
+
w_doc = anonymize_pdf_document_results(document_tree)
|
|
529
|
+
pdf_path.with_suffix(".extracted.txt").write_text(w_doc.text)
|
|
530
|
+
# print(f"Input text: {w_doc.text[:2000]}")
|
|
531
|
+
|
|
532
|
+
doc = pipeline(w_doc.text)
|
|
533
|
+
# for e in [e for e in doc.analysis.entities if e.uri in ["Person", "Company"]]:
|
|
534
|
+
# print(f"{e.begin_offset} {e.uri} {e.literal}")
|
|
535
|
+
doc = self.anonymizer(doc)
|
|
536
|
+
|
|
537
|
+
# print("doc", doc.analysis)
|
|
538
|
+
results = doc.results("wowool_anonymizer")
|
|
539
|
+
locations = results["locations"]
|
|
540
|
+
# print("locations", *locations)
|
|
541
|
+
idx = 0
|
|
542
|
+
prev_w_block = None
|
|
543
|
+
block_diff_offset = 0
|
|
544
|
+
leftover = 0
|
|
545
|
+
|
|
546
|
+
for loc in locations:
|
|
547
|
+
bo = loc[KW_BYTE_BEGIN_OFFSET]
|
|
548
|
+
idx, w_block = find_next_block(w_doc.blocks, idx, bo)
|
|
549
|
+
|
|
550
|
+
if prev_w_block is not None and prev_w_block != w_block:
|
|
551
|
+
block_diff_offset = 0
|
|
552
|
+
|
|
553
|
+
if w_block:
|
|
554
|
+
# print(w_block)
|
|
555
|
+
cur_text = w_block.block.text
|
|
556
|
+
cur_text_len = len(cur_text)
|
|
557
|
+
# print(f"{len(cur_text)=} {cur_text=}")
|
|
558
|
+
begin = loc[KW_BYTE_BEGIN_OFFSET]
|
|
559
|
+
end = loc[KW_BYTE_END_OFFSET]
|
|
560
|
+
pos_begin_in_text = (
|
|
561
|
+
begin - w_block.begin_offset + block_diff_offset - leftover
|
|
562
|
+
)
|
|
563
|
+
pos_end_in_text = (
|
|
564
|
+
end - w_block.begin_offset + block_diff_offset - leftover
|
|
565
|
+
)
|
|
566
|
+
if pos_end_in_text > cur_text_len:
|
|
567
|
+
leftover = pos_end_in_text - cur_text_len
|
|
568
|
+
pos_end_in_text = cur_text_len
|
|
569
|
+
if idx + 1 < len(w_doc.blocks):
|
|
570
|
+
next_block = w_doc.blocks[idx + 1]
|
|
571
|
+
# next_block.block.text = (
|
|
572
|
+
# " " * leftover
|
|
573
|
+
# ) + next_block.block.text[leftover:]
|
|
574
|
+
next_block.block.text = next_block.block.text[leftover:]
|
|
575
|
+
else:
|
|
576
|
+
leftover = 0
|
|
577
|
+
|
|
578
|
+
new_text = cur_text[:pos_begin_in_text]
|
|
579
|
+
replacement_text = loc[KW_ANONYMIZED]
|
|
580
|
+
if loc[KW_URI] == "Person":
|
|
581
|
+
parts = replacement_text.split("-")
|
|
582
|
+
if len(parts) == 3:
|
|
583
|
+
if parts[1] == "None":
|
|
584
|
+
replacement_text = f"{parts[0]}-{parts[2]}"
|
|
585
|
+
else:
|
|
586
|
+
parts[1] = parts[1].capitalize()
|
|
587
|
+
replacement_text = "#" + "-".join(parts[1:])
|
|
588
|
+
else:
|
|
589
|
+
replacement_text = loc[KW_ANONYMIZED]
|
|
590
|
+
|
|
591
|
+
new_text += replacement_text
|
|
592
|
+
new_text += cur_text[pos_end_in_text:]
|
|
593
|
+
# print(f"{len(new_text)=} {new_text=}")
|
|
594
|
+
w_block.block.text = new_text
|
|
595
|
+
block_diff_offset += len(replacement_text) - (end - begin)
|
|
596
|
+
|
|
597
|
+
prev_w_block = w_block
|
|
598
|
+
|
|
599
|
+
for page_number, page in enumerate(document_tree["document"].pages):
|
|
600
|
+
writer.draw_text_blocks(page.layout_blocks)
|
|
601
|
+
writer.draw_images(page.layout_blocks)
|
|
602
|
+
writer.new_page()
|
|
603
|
+
|
|
604
|
+
writer.save()
|
|
605
|
+
return output_filename
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
from wowool.document import DocumentInterface
|
|
2
|
+
from wowool.sdk import Pipeline
|
|
3
|
+
from wowool.anonymizer.core.anonymizer import Anonymizer, DefaultWriter
|
|
4
|
+
from sys import stderr
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
from wowool.tools.anonymizer.writer_config import WriterConfig
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class Writer:
|
|
10
|
+
def __init__(self, writer_config: WriterConfig) -> Path:
|
|
11
|
+
self.writer_config = writer_config
|
|
12
|
+
self.suffix = "suffix" if writer_config.suffix is None else writer_config.suffix
|
|
13
|
+
self.writer = DefaultWriter(
|
|
14
|
+
writer_config.pseudonyms,
|
|
15
|
+
writer_config.formatters,
|
|
16
|
+
writer_config.fake_locale,
|
|
17
|
+
)
|
|
18
|
+
self.anonymizer = Anonymizer(
|
|
19
|
+
writer_config.annotations,
|
|
20
|
+
self.writer,
|
|
21
|
+
)
|
|
22
|
+
|
|
23
|
+
def __call__(self, document: DocumentInterface, pipeline: Pipeline):
|
|
24
|
+
writer = self.writer
|
|
25
|
+
anonymizer_ = self.anonymizer
|
|
26
|
+
writer.set_input_data(document.data)
|
|
27
|
+
|
|
28
|
+
document = pipeline(document)
|
|
29
|
+
document = anonymizer_(document)
|
|
30
|
+
if document.has_diagnostics():
|
|
31
|
+
from wowool.utility.diagnostics import print_diagnostics
|
|
32
|
+
|
|
33
|
+
print_diagnostics(document, file=stderr)
|
|
34
|
+
else:
|
|
35
|
+
input_file = Path(document.id)
|
|
36
|
+
output_file = input_file.with_suffix(f"{self.suffix}{input_file.suffix}")
|
|
37
|
+
with open(output_file, "wb") as f:
|
|
38
|
+
f.write(writer.get_bytes())
|
|
39
|
+
return output_file
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
CONFIG = {
|
|
2
|
+
"short_description": "The Anonymizer application anonymizes concepts in a document.",
|
|
3
|
+
"long_description": "The Anonymizer application anonymizes concepts in a document. Anonymization is the process of removing or obfuscating personal information from a document. This is useful when you want to share a document with others but want to protect the privacy of the people mentioned in the document.",
|
|
4
|
+
}
|