radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
from collections import defaultdict
|
|
2
|
+
from typing import List
|
|
3
|
+
|
|
4
|
+
import tqdm
|
|
5
|
+
|
|
6
|
+
OPERATORS = {
|
|
7
|
+
">": {"func": lambda x, y: x > y, "priority": 0},
|
|
8
|
+
"<": {"func": lambda x, y: x < y, "priority": 0},
|
|
9
|
+
">=": {"func": lambda x, y: x >= y, "priority": 0},
|
|
10
|
+
"<=": {"func": lambda x, y: x <= y, "priority": 0},
|
|
11
|
+
"==": {"func": lambda x, y: x == y, "priority": 0},
|
|
12
|
+
"and": {"func": lambda x, y: x and y, "priority": 1},
|
|
13
|
+
"or": {"func": lambda x, y: x or y, "priority": 2},
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
class ApplicatorNode:
|
|
18
|
+
NODE_TYPES = ["operator", "accessor", "value"]
|
|
19
|
+
|
|
20
|
+
def __init__(self, type) -> None:
|
|
21
|
+
assert type in self.NODE_TYPES
|
|
22
|
+
self.type = type
|
|
23
|
+
|
|
24
|
+
self.value = None
|
|
25
|
+
self.child_a: ApplicatorNode = None
|
|
26
|
+
self.child_b: ApplicatorNode = None
|
|
27
|
+
self.operator_method = None
|
|
28
|
+
self.access_method = None
|
|
29
|
+
self.str_representation = None
|
|
30
|
+
|
|
31
|
+
def _evaluate(self, data):
|
|
32
|
+
if self.type == "accessor":
|
|
33
|
+
return self._access(data)
|
|
34
|
+
elif self.type == "operator":
|
|
35
|
+
return self._operation(data)
|
|
36
|
+
elif self.type == "value":
|
|
37
|
+
return self._get_value()
|
|
38
|
+
|
|
39
|
+
def _access(self, data):
|
|
40
|
+
return self.access_method(data)
|
|
41
|
+
|
|
42
|
+
def _operation(self, data):
|
|
43
|
+
return self.operator_method(
|
|
44
|
+
self.child_a._evaluate(data), self.child_b._evaluate(data)
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
def _get_value(self):
|
|
48
|
+
return self.value
|
|
49
|
+
|
|
50
|
+
def __str__(self) -> str:
|
|
51
|
+
return f"{self.type} [{self.str_representation}]"
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class PlacementAccessor:
|
|
55
|
+
WHERE_KEYS = ["beginning", "end"]
|
|
56
|
+
|
|
57
|
+
def __init__(self, object: dict, class_label_field_name: str = "labels") -> None:
|
|
58
|
+
self.class_label_field_name = class_label_field_name
|
|
59
|
+
self.object = object
|
|
60
|
+
|
|
61
|
+
def set_value(self, placement_str: str, value):
|
|
62
|
+
access_keys = placement_str.split("::")
|
|
63
|
+
where = access_keys[-1]
|
|
64
|
+
assert len(access_keys) > 1
|
|
65
|
+
assert where in self.WHERE_KEYS
|
|
66
|
+
first_key = access_keys[0]
|
|
67
|
+
object_ptr = self.object
|
|
68
|
+
accessed_data = []
|
|
69
|
+
if first_key == "text":
|
|
70
|
+
assert access_keys[1] == where
|
|
71
|
+
accessed_data.append((object_ptr, "text"))
|
|
72
|
+
elif first_key == self.class_label_field_name:
|
|
73
|
+
assert access_keys[1] == where
|
|
74
|
+
object_ptr = object_ptr[self.class_label_field_name]
|
|
75
|
+
for idx in range(len(object_ptr)):
|
|
76
|
+
accessed_data.append((object_ptr, idx))
|
|
77
|
+
else:
|
|
78
|
+
object_ptr = object_ptr["metadata"]
|
|
79
|
+
for key in access_keys[:-2]:
|
|
80
|
+
object_ptr = object_ptr[key]
|
|
81
|
+
last_ptr = object_ptr[access_keys[-2]]
|
|
82
|
+
if isinstance(last_ptr, list):
|
|
83
|
+
object_ptr = last_ptr
|
|
84
|
+
for idx in range(len(object_ptr)):
|
|
85
|
+
accessed_data.append((object_ptr, idx))
|
|
86
|
+
else:
|
|
87
|
+
accessed_data.append((object_ptr, access_keys[-2]))
|
|
88
|
+
|
|
89
|
+
for ptr, key in accessed_data:
|
|
90
|
+
ptr[key] = self._modify_string(ptr[key], value, where)
|
|
91
|
+
|
|
92
|
+
@staticmethod
|
|
93
|
+
def _modify_string(string: str, value: str, where: str):
|
|
94
|
+
if where == "beginning":
|
|
95
|
+
return value + string
|
|
96
|
+
elif where == "end":
|
|
97
|
+
return string + value
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class ApplicatorTree:
|
|
101
|
+
def __init__(
|
|
102
|
+
self,
|
|
103
|
+
root: ApplicatorNode,
|
|
104
|
+
rule_str: str,
|
|
105
|
+
placement: str,
|
|
106
|
+
special_token: str,
|
|
107
|
+
) -> None:
|
|
108
|
+
self.root: ApplicatorNode = root
|
|
109
|
+
if self.root.type != "operator":
|
|
110
|
+
raise ValueError(f"The following rule contains no operators: {rule_str}")
|
|
111
|
+
self.placement: str = placement
|
|
112
|
+
self.special_token: str = special_token
|
|
113
|
+
|
|
114
|
+
def _evaluate(self, data):
|
|
115
|
+
return self.root._evaluate(data)
|
|
116
|
+
|
|
117
|
+
def print_tree(self):
|
|
118
|
+
print(f"{self.__class__.__name__}")
|
|
119
|
+
print(str(self))
|
|
120
|
+
|
|
121
|
+
def _get_str_representation(self):
|
|
122
|
+
return self._get_str_representation_recursive(self.root, 0)
|
|
123
|
+
|
|
124
|
+
def _get_str_representation_recursive(self, node, depth):
|
|
125
|
+
if node is not None:
|
|
126
|
+
string = "\t" * depth + str(node)
|
|
127
|
+
if node.child_a is not None:
|
|
128
|
+
string += "\n" + self._get_str_representation_recursive(
|
|
129
|
+
node.child_a, depth + 1
|
|
130
|
+
)
|
|
131
|
+
if node.child_b is not None:
|
|
132
|
+
string += "\n" + self._get_str_representation_recursive(
|
|
133
|
+
node.child_b, depth + 1
|
|
134
|
+
)
|
|
135
|
+
return string
|
|
136
|
+
return ""
|
|
137
|
+
|
|
138
|
+
def __str__(self) -> str:
|
|
139
|
+
return self._get_str_representation()
|
|
140
|
+
|
|
141
|
+
def __call__(self, *args, **kwargs):
|
|
142
|
+
return self._evaluate(*args, **kwargs)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class RuleApplicator:
|
|
146
|
+
def __init__(
|
|
147
|
+
self, rule_trees: List[ApplicatorTree], special_tokens: List[str]
|
|
148
|
+
) -> None:
|
|
149
|
+
self.rule_trees: List[ApplicatorTree] = rule_trees
|
|
150
|
+
self.special_tokens: List[str] = special_tokens
|
|
151
|
+
|
|
152
|
+
def _process(self, data):
|
|
153
|
+
for elem in tqdm.tqdm(data):
|
|
154
|
+
true_trees = [tree for tree in self.rule_trees if tree(elem)]
|
|
155
|
+
self._apply_trees(true_trees, elem)
|
|
156
|
+
|
|
157
|
+
@staticmethod
|
|
158
|
+
def _apply_trees(trees: List[ApplicatorTree], elem: dict):
|
|
159
|
+
accessor = PlacementAccessor(elem)
|
|
160
|
+
placement_buckets = defaultdict(list)
|
|
161
|
+
all_placements = [tree.placement for tree in trees]
|
|
162
|
+
all_special_tokens = [tree.special_token for tree in trees]
|
|
163
|
+
for placement, special_token in zip(all_placements, all_special_tokens):
|
|
164
|
+
placement_buckets[placement].append(special_token)
|
|
165
|
+
for placement, bucket_values in placement_buckets.items():
|
|
166
|
+
tokens_fused = "".join(bucket_values)
|
|
167
|
+
accessor.set_value(placement, tokens_fused)
|
|
168
|
+
|
|
169
|
+
def __call__(self, data):
|
|
170
|
+
return self._process(data)
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
class SpecialTokensRulesParser:
|
|
174
|
+
def __init__(self, special_tokens) -> None:
|
|
175
|
+
self.special_tokens = special_tokens
|
|
176
|
+
|
|
177
|
+
def _merge_rules(self, special_token_rules):
|
|
178
|
+
groups = defaultdict(list)
|
|
179
|
+
for rule in special_token_rules:
|
|
180
|
+
group_unique_id = rule["token"] + " " + rule["placement"]
|
|
181
|
+
groups[group_unique_id].append(rule)
|
|
182
|
+
new_rules = []
|
|
183
|
+
for group in groups.values():
|
|
184
|
+
if len(group) == 1:
|
|
185
|
+
rule = group[0]
|
|
186
|
+
else:
|
|
187
|
+
when = " or ".join(f"({s['when']})" for s in group)
|
|
188
|
+
rule = {
|
|
189
|
+
"token": group[0]["token"],
|
|
190
|
+
"placement": group[0]["placement"],
|
|
191
|
+
"when": when,
|
|
192
|
+
}
|
|
193
|
+
new_rules.append(rule)
|
|
194
|
+
return new_rules
|
|
195
|
+
|
|
196
|
+
def _sort_rules(self, special_tokens_rules):
|
|
197
|
+
buckets = {k: [] for k in self.special_tokens} # ordered dict (Python 3.7)
|
|
198
|
+
for rule in special_tokens_rules:
|
|
199
|
+
token = rule["token"]
|
|
200
|
+
if token not in buckets:
|
|
201
|
+
raise KeyError(
|
|
202
|
+
f"Special token {token} used in the rule "
|
|
203
|
+
f"{rule['when']!r} is not in the list of declared "
|
|
204
|
+
f"special tokens: {self.special_tokens}."
|
|
205
|
+
)
|
|
206
|
+
buckets[token].append(rule)
|
|
207
|
+
new_rules = []
|
|
208
|
+
for bucket in buckets.values():
|
|
209
|
+
new_rules.extend(bucket)
|
|
210
|
+
return new_rules
|
|
211
|
+
|
|
212
|
+
def parse(self, special_tokens_rules) -> List[ApplicatorTree]:
|
|
213
|
+
special_tokens_rules = self._merge_rules(special_tokens_rules)
|
|
214
|
+
special_tokens_rules = self._sort_rules(special_tokens_rules)
|
|
215
|
+
|
|
216
|
+
rule_trees: List[ApplicatorTree] = []
|
|
217
|
+
for rule in special_tokens_rules:
|
|
218
|
+
token = rule["token"]
|
|
219
|
+
when = rule["when"]
|
|
220
|
+
placement = rule["placement"]
|
|
221
|
+
|
|
222
|
+
when_words = list(
|
|
223
|
+
enumerate(when.replace("(", " ( ").replace(")", " ) ").split())
|
|
224
|
+
) # whitespace split
|
|
225
|
+
|
|
226
|
+
root_node = self.parse_when_words(when_words, when)
|
|
227
|
+
when_tree = ApplicatorTree(
|
|
228
|
+
root=root_node,
|
|
229
|
+
rule_str=when,
|
|
230
|
+
placement=placement,
|
|
231
|
+
special_token=token,
|
|
232
|
+
)
|
|
233
|
+
rule_trees.append(when_tree)
|
|
234
|
+
return rule_trees
|
|
235
|
+
|
|
236
|
+
def parse_when_words(self, when_words, sentence):
|
|
237
|
+
try:
|
|
238
|
+
return self._parse_when_words_recursive(when_words)
|
|
239
|
+
except (AssertionError, KeyError, IndexError, ValueError) as error:
|
|
240
|
+
raise Exception(
|
|
241
|
+
f"Error has occured while parsing the following rule: {sentence}"
|
|
242
|
+
) from error
|
|
243
|
+
|
|
244
|
+
def _are_outer_brackets_present(self, expression):
|
|
245
|
+
if (
|
|
246
|
+
len(expression) >= 3
|
|
247
|
+
and expression[0][1] == "("
|
|
248
|
+
and expression[-1][1] == ")"
|
|
249
|
+
):
|
|
250
|
+
stack = 0
|
|
251
|
+
for _, word in expression[1:-1]:
|
|
252
|
+
if word == "(":
|
|
253
|
+
stack += 1
|
|
254
|
+
elif word == ")":
|
|
255
|
+
if stack == 0:
|
|
256
|
+
return False
|
|
257
|
+
stack -= 1
|
|
258
|
+
return stack == 0
|
|
259
|
+
return False
|
|
260
|
+
|
|
261
|
+
def _parse_when_words_recursive(
|
|
262
|
+
self, when_words: list, used_operators: list = None
|
|
263
|
+
) -> ApplicatorNode:
|
|
264
|
+
if self._are_outer_brackets_present(when_words):
|
|
265
|
+
when_words = when_words[1:-1]
|
|
266
|
+
|
|
267
|
+
used_operators = []
|
|
268
|
+
bracket_count = 0
|
|
269
|
+
for idx, word in when_words:
|
|
270
|
+
if word == "(":
|
|
271
|
+
bracket_count += 1
|
|
272
|
+
elif word == ")":
|
|
273
|
+
bracket_count -= 1
|
|
274
|
+
elif word in OPERATORS and bracket_count == 0:
|
|
275
|
+
operator = OPERATORS[word]
|
|
276
|
+
used_operators.append(
|
|
277
|
+
{"word_idx": idx, "operator": operator, "word": word}
|
|
278
|
+
)
|
|
279
|
+
|
|
280
|
+
if len(used_operators) == 0:
|
|
281
|
+
assert len(when_words) == 1
|
|
282
|
+
word = when_words[0][
|
|
283
|
+
1
|
|
284
|
+
] # 0 because first word in the list, and 1 because index 0 is the index of a word and index 1 is the string
|
|
285
|
+
|
|
286
|
+
is_value = word.replace(".", "").isnumeric()
|
|
287
|
+
if is_value:
|
|
288
|
+
node = ApplicatorNode(type="value")
|
|
289
|
+
node.value = float(word)
|
|
290
|
+
node.str_representation = word
|
|
291
|
+
return node
|
|
292
|
+
|
|
293
|
+
access_keys = word.split("::")
|
|
294
|
+
|
|
295
|
+
def access_method(x):
|
|
296
|
+
first_key = access_keys[0]
|
|
297
|
+
if first_key not in ["text", "labels"]:
|
|
298
|
+
x = x["metadata"]
|
|
299
|
+
for access_key in access_keys:
|
|
300
|
+
x = x[access_key]
|
|
301
|
+
return x
|
|
302
|
+
|
|
303
|
+
node = ApplicatorNode(type="accessor")
|
|
304
|
+
node.access_method = access_method
|
|
305
|
+
node.str_representation = word
|
|
306
|
+
return node
|
|
307
|
+
else:
|
|
308
|
+
active_operator = max(
|
|
309
|
+
used_operators,
|
|
310
|
+
key=lambda x: (x["operator"]["priority"], x["word_idx"]),
|
|
311
|
+
)
|
|
312
|
+
split_point = active_operator["word_idx"]
|
|
313
|
+
words_left = [w for w in when_words if w[0] < split_point]
|
|
314
|
+
words_right = [w for w in when_words if w[0] > split_point]
|
|
315
|
+
assert len(words_left) > 0
|
|
316
|
+
assert len(words_right) > 0
|
|
317
|
+
operators_left = [
|
|
318
|
+
op for op in used_operators if op["word_idx"] < split_point
|
|
319
|
+
]
|
|
320
|
+
operators_right = [
|
|
321
|
+
op for op in used_operators if op["word_idx"] > split_point
|
|
322
|
+
]
|
|
323
|
+
node = ApplicatorNode(type="operator")
|
|
324
|
+
node.child_a = self._parse_when_words_recursive(words_left, operators_left)
|
|
325
|
+
node.child_b = self._parse_when_words_recursive(
|
|
326
|
+
words_right, operators_right
|
|
327
|
+
)
|
|
328
|
+
node.operator_method = active_operator["operator"]["func"]
|
|
329
|
+
node.str_representation = f"{active_operator['word']}"
|
|
330
|
+
return node
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import threading
|
|
2
|
+
|
|
3
|
+
from typing import List
|
|
4
|
+
from abc import ABC, abstractmethod
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class ThreadWorker(ABC):
|
|
8
|
+
def __init__(self, debug: bool = False) -> None:
|
|
9
|
+
self._debug = debug
|
|
10
|
+
self._thread = threading.Thread(target=self._task)
|
|
11
|
+
self._enabled = True
|
|
12
|
+
|
|
13
|
+
def start(self) -> None:
|
|
14
|
+
if not self._enabled:
|
|
15
|
+
raise Exception("Cannot run again a disabled task.")
|
|
16
|
+
if self._debug:
|
|
17
|
+
print(f"Task running {self.__class__.__name__}")
|
|
18
|
+
self._thread.start()
|
|
19
|
+
|
|
20
|
+
@abstractmethod
|
|
21
|
+
def _task(self) -> None:
|
|
22
|
+
pass
|
|
23
|
+
|
|
24
|
+
def disable(self):
|
|
25
|
+
if self._debug:
|
|
26
|
+
print(f"Task disabled {self.__class__.__name__}")
|
|
27
|
+
self._enabled = False
|
|
28
|
+
|
|
29
|
+
def join(self):
|
|
30
|
+
self._thread.join()
|
|
31
|
+
if self._debug:
|
|
32
|
+
print(f"Task finished {self.__class__.__name__}")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class WorkerCluster:
|
|
36
|
+
def __init__(
|
|
37
|
+
self, name: str, threads: List[ThreadWorker], debug: bool = False
|
|
38
|
+
) -> None:
|
|
39
|
+
self._debug = debug
|
|
40
|
+
self._name = self.__class__.__name__
|
|
41
|
+
if name is not None:
|
|
42
|
+
self._name = name
|
|
43
|
+
self._threads = threads
|
|
44
|
+
|
|
45
|
+
def start(self):
|
|
46
|
+
if self._debug:
|
|
47
|
+
print(f"Running cluster {self._name}")
|
|
48
|
+
for thread in self._threads:
|
|
49
|
+
thread.start()
|
|
50
|
+
|
|
51
|
+
def disable(self):
|
|
52
|
+
if self._debug:
|
|
53
|
+
print(f"Disabling cluster {self._name}")
|
|
54
|
+
for thread in self._threads:
|
|
55
|
+
thread.disable()
|
|
56
|
+
|
|
57
|
+
def join(self):
|
|
58
|
+
if self._debug:
|
|
59
|
+
print(f"Joining cluster {self._name}")
|
|
60
|
+
for thread in self._threads:
|
|
61
|
+
thread.join()
|
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: radlab-data
|
|
3
|
+
Version: 0.0.4
|
|
4
|
+
Summary: Text processing and modelling
|
|
5
|
+
Author-email: RadLab <hello@radlab.dev>
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://radlab.dev
|
|
8
|
+
Classifier: Development Status :: 4 - Beta
|
|
9
|
+
Classifier: Intended Audience :: Developers
|
|
10
|
+
Classifier: Intended Audience :: Science/Research
|
|
11
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Text Processing
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: tqdm
|
|
24
|
+
Requires-Dist: requests
|
|
25
|
+
Requires-Dist: beautifulsoup4
|
|
26
|
+
Requires-Dist: python-docx
|
|
27
|
+
Requires-Dist: pypdf
|
|
28
|
+
Provides-Extra: ml
|
|
29
|
+
Requires-Dist: transformers; extra == "ml"
|
|
30
|
+
Requires-Dist: torch; extra == "ml"
|
|
31
|
+
Requires-Dist: datasets; extra == "ml"
|
|
32
|
+
Requires-Dist: numpy; extra == "ml"
|
|
33
|
+
Requires-Dist: scikit-learn; extra == "ml"
|
|
34
|
+
Requires-Dist: scipy; extra == "ml"
|
|
35
|
+
Requires-Dist: matplotlib; extra == "ml"
|
|
36
|
+
Provides-Extra: spacy
|
|
37
|
+
Requires-Dist: spacy; extra == "spacy"
|
|
38
|
+
Provides-Extra: lang
|
|
39
|
+
Requires-Dist: fasttext-langdetect; extra == "lang"
|
|
40
|
+
Requires-Dist: deepl; extra == "lang"
|
|
41
|
+
Provides-Extra: tables
|
|
42
|
+
Requires-Dist: pandas; extra == "tables"
|
|
43
|
+
Requires-Dist: openpyxl; extra == "tables"
|
|
44
|
+
Provides-Extra: all
|
|
45
|
+
Requires-Dist: transformers; extra == "all"
|
|
46
|
+
Requires-Dist: torch; extra == "all"
|
|
47
|
+
Requires-Dist: datasets; extra == "all"
|
|
48
|
+
Requires-Dist: numpy; extra == "all"
|
|
49
|
+
Requires-Dist: scikit-learn; extra == "all"
|
|
50
|
+
Requires-Dist: scipy; extra == "all"
|
|
51
|
+
Requires-Dist: matplotlib; extra == "all"
|
|
52
|
+
Requires-Dist: spacy; extra == "all"
|
|
53
|
+
Requires-Dist: fasttext-langdetect; extra == "all"
|
|
54
|
+
Requires-Dist: deepl; extra == "all"
|
|
55
|
+
Requires-Dist: pandas; extra == "all"
|
|
56
|
+
Requires-Dist: openpyxl; extra == "all"
|
|
57
|
+
Dynamic: license-file
|
|
58
|
+
|
|
59
|
+
# radlab‑data
|
|
60
|
+
|
|
61
|
+
A versatile Python library for loading, preprocessing, and handling a variety of textual data sources (PDF, DOCX, TXT, MD, HTML,
|
|
62
|
+
and raw strings) and preparing them for downstream machine‑learning tasks such as classification, sequence labeling, and
|
|
63
|
+
generative modeling.
|
|
64
|
+
|
|
65
|
+
---
|
|
66
|
+
|
|
67
|
+
## Table of Contents
|
|
68
|
+
|
|
69
|
+
- [Overview](#overview)
|
|
70
|
+
- [Features](#features)
|
|
71
|
+
- [Installation](#installation)
|
|
72
|
+
- [Quick Start](#quick-start)
|
|
73
|
+
- [Core Concepts](#core-concepts)
|
|
74
|
+
- [Dataset Types](#dataset-types)
|
|
75
|
+
- [Text Loaders](#text-loaders)
|
|
76
|
+
- [Processing Pipeline](#processing-pipeline)
|
|
77
|
+
- [Command‑Line Utilities](#command-line-utilities)
|
|
78
|
+
- [Configuration](#configuration)
|
|
79
|
+
- [License](#license)
|
|
80
|
+
|
|
81
|
+
---
|
|
82
|
+
|
|
83
|
+
## Overview
|
|
84
|
+
|
|
85
|
+
`radlab-data` provides a unified interface for:
|
|
86
|
+
|
|
87
|
+
1. **Reading documents** from common file formats (PDF, DOCX, TXT, MD, HTML) as well as plain‑text strings.
|
|
88
|
+
2. **Cleaning and normalising** raw text using a chain of configurable processors.
|
|
89
|
+
3. **Splitting** text into sentences, paragraphs, or token‑level chunks.
|
|
90
|
+
4. **Mapping** raw annotations (e.g., NER spans) onto tokenised representations.
|
|
91
|
+
5. **Packaging** data into HuggingFace `datasets.Dataset` objects, ready for training with `transformers`.
|
|
92
|
+
|
|
93
|
+
The library is built with extensibility in mind – you can plug in your own tokenizers, preprocessing steps, or custom
|
|
94
|
+
dataset classes.
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Features
|
|
99
|
+
|
|
100
|
+
- **Multi‑format document loading** – PDF, DOCX, TXT, MD, HTML, and custom “input_text”.
|
|
101
|
+
- **Automatic language detection** (via `ftlangdetect`).
|
|
102
|
+
- **Flexible text cleaning** – phrase removal, abbreviation normalisation, regex‑based sanitisation, etc.
|
|
103
|
+
- **Sentence‑level splitting** using spaCy’s Polish model (`pl_core_news_lg`).
|
|
104
|
+
- **Special token handling** – conditional insertion of tokens based on metadata rules.
|
|
105
|
+
- **Dataset abstractions** for:
|
|
106
|
+
- Text classification
|
|
107
|
+
- Sequence labeling (NER, IOB tagging)
|
|
108
|
+
- Sequence‑to‑sequence tasks (e.g., translation, summarisation)
|
|
109
|
+
- Semantic similarity and dialogue alternatives
|
|
110
|
+
- **Caching** of tokenised datasets for fast reloads.
|
|
111
|
+
- **Multiprocessing support** for loading large corpora.
|
|
112
|
+
- **Command‑line tools** for dataset conversion, preprocessing, and training argument handling.
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Installation
|
|
117
|
+
|
|
118
|
+
```shell script
|
|
119
|
+
# Create a virtual environment (optional but recommended)
|
|
120
|
+
python -m venv .venv
|
|
121
|
+
source .venv/bin/activate # on Windows: .venv\Scripts\activate
|
|
122
|
+
|
|
123
|
+
# Core only: document loading, cleaning and small utilities
|
|
124
|
+
pip install -e .
|
|
125
|
+
|
|
126
|
+
# Everything, including the model stack
|
|
127
|
+
pip install -e ".[all]"
|
|
128
|
+
```
|
|
129
|
+
|
|
130
|
+
> **Note**: The library requires Python 3.10+.
|
|
131
|
+
|
|
132
|
+
### Dependency extras
|
|
133
|
+
|
|
134
|
+
The core installs only the light packages needed to read and clean documents
|
|
135
|
+
(`tqdm`, `requests`, `beautifulsoup4`, `python-docx`, `pypdf`). Importing the
|
|
136
|
+
document loaders, `Document`, `DirectoryFileReader` or `TextUtils` therefore
|
|
137
|
+
does **not** pull in `torch`, `transformers` or `spacy` — a query-only host
|
|
138
|
+
(for example the [Semantic Search Engine](https://dev.radlab.pl/semantic-search-engine) API)
|
|
139
|
+
can install the core alone.
|
|
140
|
+
|
|
141
|
+
| Extra | Installs | Needed for |
|
|
142
|
+
|------------|-------------------------------------------------------------------|------------------------------------------------------------------|
|
|
143
|
+
| `ml` | `transformers`, `torch`, `datasets`, `numpy`, `scikit-learn`, `scipy`, `matplotlib` | token-level chunking (`max_tokens_in_chunk`), HF datasets, metrics and training helpers |
|
|
144
|
+
| `spacy` | `spacy` | `SentenceSplitter` and `PairReader` (Polish model: `python -m spacy download pl_core_news_lg`) |
|
|
145
|
+
| `lang` | `fasttext-langdetect`, `deepl` | `TextUtils.text_language`, `TextUtils.translate_text_deepl` |
|
|
146
|
+
| `tables` | `pandas`, `openpyxl` | dataset preprocessing and spreadsheet input |
|
|
147
|
+
| `all` | all of the above | the full feature set |
|
|
148
|
+
|
|
149
|
+
Heavy imports are lazy: `transformers` is imported inside
|
|
150
|
+
`radlab_data.text.loaders.loader_i.get_tokenizer()` (so inside the first use of
|
|
151
|
+
token chunking), `spacy` inside
|
|
152
|
+
`radlab_data.text.processors.splitter.get_nlp()`, and `ftlangdetect`/`deepl`
|
|
153
|
+
inside the `TextUtils` methods which use them. `requirements.txt` lists every
|
|
154
|
+
extra for a full development environment.
|
|
155
|
+
|
|
156
|
+
---
|
|
157
|
+
|
|
158
|
+
## Quick Start
|
|
159
|
+
|
|
160
|
+
```python
|
|
161
|
+
from radlab_data.text.document import Document
|
|
162
|
+
from radlab_data.datasets.dataset_loader import DatasetLoader
|
|
163
|
+
from transformers import AutoTokenizer
|
|
164
|
+
|
|
165
|
+
# 1️⃣ Load a document (PDF, DOCX, TXT, MD, HTML, or raw text)
|
|
166
|
+
doc = Document(
|
|
167
|
+
file_path="data/example.pdf",
|
|
168
|
+
prepare_proper_pages=True,
|
|
169
|
+
clear_texts=True,
|
|
170
|
+
use_text_denoiser=False,
|
|
171
|
+
)
|
|
172
|
+
pages = doc.load() # pages is a list of `DocumentPage` objects
|
|
173
|
+
|
|
174
|
+
# 2️⃣ Initialise a tokenizer
|
|
175
|
+
tokenizer = AutoTokenizer.from_pretrained("radlab/polish-fast-tokenizer")
|
|
176
|
+
|
|
177
|
+
# 3️⃣ Load a dataset (JSON or JSONL) and automatically cache tokenisation
|
|
178
|
+
loader = DatasetLoader(
|
|
179
|
+
dataset_path="data/my_dataset",
|
|
180
|
+
tokenizer=tokenizer,
|
|
181
|
+
split_train_valid_test="0.8:0.1", # 80 % train, 10 % validation, 10 % test
|
|
182
|
+
pre_shuffle=True,
|
|
183
|
+
)
|
|
184
|
+
dataset = loader.load()
|
|
185
|
+
|
|
186
|
+
# 4️⃣ Access train / validation / test splits
|
|
187
|
+
train = dataset.train_dataset
|
|
188
|
+
valid = dataset.valid_dataset
|
|
189
|
+
test = dataset.test_dataset
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The resulting `train`, `valid`, and `test` objects are HuggingFace `datasets.Dataset` instances, ready to be passed to a
|
|
193
|
+
`transformers` trainer.
|
|
194
|
+
|
|
195
|
+
---
|
|
196
|
+
|
|
197
|
+
## Core Concepts
|
|
198
|
+
|
|
199
|
+
### Dataset Types
|
|
200
|
+
|
|
201
|
+
| Class | Typical Use‑Case | Key Characteristics |
|
|
202
|
+
|-------------------------------|-------------------------------------------------|-------------------------------------------------------------------|
|
|
203
|
+
| `TextClassificationDataset` | Single‑label classification | Stores `text` and a scalar `label`. |
|
|
204
|
+
| `SequenceLabellingDataset` | NER / token‑level tagging | Generates IOB tags aligned to tokenised inputs. |
|
|
205
|
+
| `SequenceToSequenceDataset` | Translation, summarisation, etc. | Handles source‑target pairs (`text` ↔ `label`). |
|
|
206
|
+
| `SemanticSimilarityDataset` | Pairwise similarity / ranking | Provides `sentence1`, `sentence2`, and a similarity `score`. |
|
|
207
|
+
| `DPODataset` | Preference‑based learning (chosen vs. rejected) | Keeps raw prompts and two possible completions. |
|
|
208
|
+
| `DialogueAlternativesDataset` | Multi‑choice dialogue generation | Stores a shared context and several possible endings with scores. |
|
|
209
|
+
|
|
210
|
+
All dataset classes inherit from `DatasetBaseClass`, which implements caching, optional shuffling, and split handling.
|
|
211
|
+
|
|
212
|
+
### Text Loaders
|
|
213
|
+
|
|
214
|
+
| Loader | Supported Extension | Behaviour |
|
|
215
|
+
|-------------------|---------------------|--------------------------------------------------------|
|
|
216
|
+
| `PDFLoader` | `.pdf` | One page per PDF page via `pypdf`. |
|
|
217
|
+
| `TXTLoader` | `.txt` | Whole file as one page, with encoding autodetection. |
|
|
218
|
+
| `MarkdownLoader` | `.md` | Reads like `TXTLoader`, optional markup stripping. |
|
|
219
|
+
| `DOCXLoader` | `.docx` | Parses paragraphs and tables via `python-docx`. |
|
|
220
|
+
| `HTMLLoader` | `.html` | Strips markup via BeautifulSoup, `split_by_paragraph`. |
|
|
221
|
+
| `InputTextLoader` | `.input_text` | Wraps a raw string supplied via options. |
|
|
222
|
+
|
|
223
|
+
Loaders share a common base (`LoaderI`) that handles:
|
|
224
|
+
|
|
225
|
+
- Tokeniser initialisation,
|
|
226
|
+
- Optional language detection,
|
|
227
|
+
- Text denoising,
|
|
228
|
+
- Chunking into token‑size windows.
|
|
229
|
+
|
|
230
|
+
Every loader returns `DocumentPage` objects (`page_content` plus a `metadata`
|
|
231
|
+
dictionary), so no document-loading framework is needed to use `radlab-data`.
|
|
232
|
+
|
|
233
|
+
Markdown files are read as plain text and every page is tagged with
|
|
234
|
+
`document_format: markdown`. Pass the `strip_markdown` option to remove markdown markup
|
|
235
|
+
(headings, emphasis, links, code fences, list markers) while keeping the underlying text:
|
|
236
|
+
|
|
237
|
+
```python
|
|
238
|
+
from radlab_data.text.loaders.markdown import MarkdownLoader
|
|
239
|
+
|
|
240
|
+
loader = MarkdownLoader("docs/README.md", options={"strip_markdown": True})
|
|
241
|
+
pages = loader.load()
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
### Processing Pipeline
|
|
245
|
+
|
|
246
|
+
`radlab-data` ships with a lightweight pipeline framework:
|
|
247
|
+
|
|
248
|
+
```python
|
|
249
|
+
from radlab_data.preprocessing.pipeline import Pipeline
|
|
250
|
+
from radlab_data.preprocessing.pipeline_modules import SplitLabels, RemoveDuplicates
|
|
251
|
+
|
|
252
|
+
pipeline = Pipeline(modules=[
|
|
253
|
+
SplitLabels(),
|
|
254
|
+
RemoveDuplicates(),
|
|
255
|
+
# add your custom PipelineElementBase subclasses here
|
|
256
|
+
])
|
|
257
|
+
|
|
258
|
+
processed_dataset = pipeline(original_dataset)
|
|
259
|
+
```
|
|
260
|
+
|
|
261
|
+
Each pipeline element inherits from `PipelineElementBase` and implements a `_process_data` method that receives a
|
|
262
|
+
dataset instance and returns the transformed instance. The framework automatically validates that the element supports
|
|
263
|
+
the dataset type.
|
|
264
|
+
|
|
265
|
+
---
|
|
266
|
+
|
|
267
|
+
## Command‑Line Utilities
|
|
268
|
+
|
|
269
|
+
The package currently does not ship command‑line entry points; dataset
|
|
270
|
+
conversion and preprocessing are driven from Python through the classes
|
|
271
|
+
described above.
|
|
272
|
+
|
|
273
|
+
---
|
|
274
|
+
|
|
275
|
+
## Configuration
|
|
276
|
+
|
|
277
|
+
All default settings live under `radlab_data/text/loaders/config.py`:
|
|
278
|
+
|
|
279
|
+
```python
|
|
280
|
+
TOKENIZER_PATH = "radlab/polish-fast-tokenizer"
|
|
281
|
+
AVAILABLE_FILE_EXTENSIONS = ["txt", "md", "pdf", "docx", "html", "input_text"]
|
|
282
|
+
```
|
|
283
|
+
|
|
284
|
+
You can override these values at runtime by passing the appropriate arguments to `Document` or the loader classes.
|
|
285
|
+
|
|
286
|
+
---
|
|
287
|
+
|
|
288
|
+
## License
|
|
289
|
+
|
|
290
|
+
`radlab-data` is released under the **Apache 2.0 License**. See the `LICENSE` file for full details.
|