radlab-data 0.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- radlab_data/__init__.py +9 -0
- radlab_data/datasets/__init__.py +0 -0
- radlab_data/datasets/dataset.py +509 -0
- radlab_data/datasets/dataset_loader.py +140 -0
- radlab_data/datasets/tokenization_handler.py +78 -0
- radlab_data/preprocessing/__init__.py +0 -0
- radlab_data/preprocessing/dataset.py +726 -0
- radlab_data/preprocessing/pipeline.py +38 -0
- radlab_data/preprocessing/pipeline_modules.py +342 -0
- radlab_data/text/__init__.py +0 -0
- radlab_data/text/document.py +136 -0
- radlab_data/text/loaders/__init__.py +0 -0
- radlab_data/text/loaders/base.py +65 -0
- radlab_data/text/loaders/config.py +4 -0
- radlab_data/text/loaders/docx.py +96 -0
- radlab_data/text/loaders/html.py +111 -0
- radlab_data/text/loaders/input_txt.py +30 -0
- radlab_data/text/loaders/loader_i.py +267 -0
- radlab_data/text/loaders/markdown.py +129 -0
- radlab_data/text/loaders/pdf.py +157 -0
- radlab_data/text/loaders/txt.py +44 -0
- radlab_data/text/pair_reader.py +241 -0
- radlab_data/text/processors/__init__.py +0 -0
- radlab_data/text/processors/base.py +23 -0
- radlab_data/text/processors/cleaner.py +338 -0
- radlab_data/text/processors/mappers.py +167 -0
- radlab_data/text/processors/regex_defs.py +58 -0
- radlab_data/text/processors/splitter.py +60 -0
- radlab_data/text/reader.py +198 -0
- radlab_data/text/utils.py +51 -0
- radlab_data/utils/__init__.py +0 -0
- radlab_data/utils/api_handler.py +83 -0
- radlab_data/utils/argument_parser.py +241 -0
- radlab_data/utils/hf_training_arguments.py +95 -0
- radlab_data/utils/metrics.py +43 -0
- radlab_data/utils/special_tokens_rules.py +330 -0
- radlab_data/utils/threads.py +61 -0
- radlab_data-0.0.4.dist-info/METADATA +290 -0
- radlab_data-0.0.4.dist-info/RECORD +42 -0
- radlab_data-0.0.4.dist-info/WHEEL +5 -0
- radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
- radlab_data-0.0.4.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
from typing import List
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
class TokenizationHandler:
|
|
5
|
+
"""
|
|
6
|
+
Handles all the tokenization (and paralellization) logic.
|
|
7
|
+
|
|
8
|
+
Algorithm details:
|
|
9
|
+
This class is used for fast dictionary tokenization.
|
|
10
|
+
Acyclic Graph Tokenizer by Kacper Schnetzer
|
|
11
|
+
Created: 2023
|
|
12
|
+
|
|
13
|
+
@ :)
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
def __init__(
|
|
17
|
+
self,
|
|
18
|
+
tokenizer,
|
|
19
|
+
blacklist: List[str] = None,
|
|
20
|
+
tokenization_batch_size: int = 256,
|
|
21
|
+
) -> None:
|
|
22
|
+
"""
|
|
23
|
+
Parameters:
|
|
24
|
+
tokenizer - a HuggingFace tokenizer class used for tokenization
|
|
25
|
+
blacklist: List[str] - if tokenizing a dict: list of keys in the dict
|
|
26
|
+
that the tokenizer pass through
|
|
27
|
+
"""
|
|
28
|
+
self._tokenizer = tokenizer
|
|
29
|
+
self._blacklist = set() if blacklist is None else set(blacklist)
|
|
30
|
+
self._tokenization_batch_size = tokenization_batch_size
|
|
31
|
+
|
|
32
|
+
def _get_gatherer(self, obj, parent=None, key=None):
|
|
33
|
+
if isinstance(obj, str):
|
|
34
|
+
yield obj, parent, key
|
|
35
|
+
elif isinstance(obj, dict):
|
|
36
|
+
for k, v in obj.items():
|
|
37
|
+
if k in self._blacklist:
|
|
38
|
+
continue
|
|
39
|
+
yield from self._get_gatherer(v, parent=obj, key=k)
|
|
40
|
+
elif isinstance(obj, list):
|
|
41
|
+
for k, v in enumerate(obj):
|
|
42
|
+
yield from self._get_gatherer(v, parent=obj, key=k)
|
|
43
|
+
|
|
44
|
+
def _gather(self, gatherer, element_count):
|
|
45
|
+
out_arr = []
|
|
46
|
+
for _ in range(element_count):
|
|
47
|
+
elem = next(gatherer, None)
|
|
48
|
+
if elem is None:
|
|
49
|
+
return out_arr, True
|
|
50
|
+
else:
|
|
51
|
+
out_arr.append(elem)
|
|
52
|
+
return out_arr, False
|
|
53
|
+
|
|
54
|
+
def _update_branch(self, parent, value, key):
|
|
55
|
+
parent[key] = value
|
|
56
|
+
|
|
57
|
+
def _update_tree(self, branches):
|
|
58
|
+
for branch in branches:
|
|
59
|
+
self._update_branch(parent=branch[1], value=branch[0], key=branch[2])
|
|
60
|
+
|
|
61
|
+
def tokenize(self, dataset):
|
|
62
|
+
element_gatherer = self._get_gatherer(dataset)
|
|
63
|
+
while True:
|
|
64
|
+
branches, finished = self._gather(
|
|
65
|
+
element_gatherer, element_count=self._tokenization_batch_size
|
|
66
|
+
)
|
|
67
|
+
if not branches:
|
|
68
|
+
break
|
|
69
|
+
leaves = [branch[0] for branch in branches]
|
|
70
|
+
tokenized_leaves = self._tokenizer(leaves)["input_ids"]
|
|
71
|
+
branches = [
|
|
72
|
+
(tokenized_leaf, branch[1], branch[2])
|
|
73
|
+
for tokenized_leaf, branch in zip(tokenized_leaves, branches)
|
|
74
|
+
]
|
|
75
|
+
self._update_tree(branches=branches)
|
|
76
|
+
if finished:
|
|
77
|
+
break
|
|
78
|
+
return dataset
|
|
File without changes
|