radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,78 @@
1
+ from typing import List
2
+
3
+
4
+ class TokenizationHandler:
5
+ """
6
+ Handles all the tokenization (and paralellization) logic.
7
+
8
+ Algorithm details:
9
+ This class is used for fast dictionary tokenization.
10
+ Acyclic Graph Tokenizer by Kacper Schnetzer
11
+ Created: 2023
12
+
13
+ @ :)
14
+ """
15
+
16
+ def __init__(
17
+ self,
18
+ tokenizer,
19
+ blacklist: List[str] = None,
20
+ tokenization_batch_size: int = 256,
21
+ ) -> None:
22
+ """
23
+ Parameters:
24
+ tokenizer - a HuggingFace tokenizer class used for tokenization
25
+ blacklist: List[str] - if tokenizing a dict: list of keys in the dict
26
+ that the tokenizer pass through
27
+ """
28
+ self._tokenizer = tokenizer
29
+ self._blacklist = set() if blacklist is None else set(blacklist)
30
+ self._tokenization_batch_size = tokenization_batch_size
31
+
32
+ def _get_gatherer(self, obj, parent=None, key=None):
33
+ if isinstance(obj, str):
34
+ yield obj, parent, key
35
+ elif isinstance(obj, dict):
36
+ for k, v in obj.items():
37
+ if k in self._blacklist:
38
+ continue
39
+ yield from self._get_gatherer(v, parent=obj, key=k)
40
+ elif isinstance(obj, list):
41
+ for k, v in enumerate(obj):
42
+ yield from self._get_gatherer(v, parent=obj, key=k)
43
+
44
+ def _gather(self, gatherer, element_count):
45
+ out_arr = []
46
+ for _ in range(element_count):
47
+ elem = next(gatherer, None)
48
+ if elem is None:
49
+ return out_arr, True
50
+ else:
51
+ out_arr.append(elem)
52
+ return out_arr, False
53
+
54
+ def _update_branch(self, parent, value, key):
55
+ parent[key] = value
56
+
57
+ def _update_tree(self, branches):
58
+ for branch in branches:
59
+ self._update_branch(parent=branch[1], value=branch[0], key=branch[2])
60
+
61
+ def tokenize(self, dataset):
62
+ element_gatherer = self._get_gatherer(dataset)
63
+ while True:
64
+ branches, finished = self._gather(
65
+ element_gatherer, element_count=self._tokenization_batch_size
66
+ )
67
+ if not branches:
68
+ break
69
+ leaves = [branch[0] for branch in branches]
70
+ tokenized_leaves = self._tokenizer(leaves)["input_ids"]
71
+ branches = [
72
+ (tokenized_leaf, branch[1], branch[2])
73
+ for tokenized_leaf, branch in zip(tokenized_leaves, branches)
74
+ ]
75
+ self._update_tree(branches=branches)
76
+ if finished:
77
+ break
78
+ return dataset
File without changes