radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,241 @@
1
+ from argparse import ArgumentParser
2
+
3
+ INPUT_FILE_REQUIRED = "input_file_required"
4
+ INPUT_FILE_NOT_REQUIRED = "input_file_not_required"
5
+ INPUT_DIR_REQUIRED = "input_dir_required"
6
+ INPUT_DIR_NOT_REQUIRED = "input_dir_not_required"
7
+ OUTPUT_FILE_REQUIRED = "output_file_required"
8
+ OUTPUT_FILE_NOT_REQUIRED = "output_file_not_required"
9
+ OUTPUT_DIR_REQUIRED = "output_dir_required"
10
+ OUTPUT_DIR_NOT_REQUIRED = "output_dir_not_required"
11
+ WORK_DIR_REQUIRED = "work_dir_required"
12
+ WORK_DIR_NOT_REQUIRED = "work_dir_not_required"
13
+ BASE_MODEL_REQUIRED = "base_model_required"
14
+ BASE_MODEL_NOT_REQUIRED = "base_model_not_required"
15
+ REWARD_MODEL_REQUIRED = "reward_model_required"
16
+ REWARD_MODEL_NOT_REQUIRED = "reward_model_not_required"
17
+ TRAIN_INPUT_FILE_REQUIRED = "train_input_file_required"
18
+ TRAIN_INPUT_FILE_NOT_REQUIRED = "train_input_file_not_required"
19
+ TEST_INPUT_FILE_REQUIRED = "test_input_file_required"
20
+ TEST_INPUT_FILE_NOT_REQUIRED = "test_input_file_not_required"
21
+ VALID_INPUT_FILE_REQUIRED = "valid_input_file_required"
22
+ VALID_INPUT_FILE_NOT_REQUIRED = "valid_input_file_not_required"
23
+ WANDB_BOOLEAN_FULL = "wandb_boolean_full"
24
+
25
+
26
+ class _APM:
27
+ """
28
+ Argument Parser Mapper
29
+ """
30
+
31
+ class APMD:
32
+ """
33
+ Argument parsr mapper definitions
34
+ """
35
+
36
+ PMAP_SHORT = "short"
37
+ PMAP_LONG = "long"
38
+ PMAP_DESCR = "description"
39
+ PMAP_DEST = "dest"
40
+ PMAP_REQ = "required"
41
+ PMAP_TYPE = "type"
42
+
43
+ INPUT_FILE = {
44
+ PMAP_SHORT: "-i",
45
+ PMAP_LONG: "--input-file",
46
+ PMAP_DEST: "input_file",
47
+ PMAP_DESCR: "Path to input file",
48
+ PMAP_TYPE: "str",
49
+ }
50
+ INPUT_DIR = {
51
+ PMAP_SHORT: "-I",
52
+ PMAP_LONG: "--input-dir",
53
+ PMAP_DEST: "input_dir",
54
+ PMAP_DESCR: "Path to input directory",
55
+ PMAP_TYPE: "str",
56
+ }
57
+ OUTPUT_FILE = {
58
+ PMAP_SHORT: "-o",
59
+ PMAP_LONG: "--output-file",
60
+ PMAP_DEST: "output_file",
61
+ PMAP_DESCR: "Output file",
62
+ PMAP_TYPE: "str",
63
+ }
64
+ OUTPUT_DIR = {
65
+ PMAP_SHORT: "-O",
66
+ PMAP_LONG: "--output-dir",
67
+ PMAP_DEST: "output_dir",
68
+ PMAP_DESCR: "Output directory",
69
+ PMAP_TYPE: "str",
70
+ }
71
+ WORK_DIR = {
72
+ PMAP_SHORT: "-W",
73
+ PMAP_LONG: "--work-dir",
74
+ PMAP_DEST: "work_dir",
75
+ PMAP_DESCR: "Path to temporary work directory",
76
+ PMAP_TYPE: "str",
77
+ }
78
+ TRAIN_INPUT_FILE = {
79
+ PMAP_SHORT: "-T",
80
+ PMAP_LONG: "--train-file",
81
+ PMAP_DEST: "train_file",
82
+ PMAP_DESCR: "Path to train input file",
83
+ PMAP_TYPE: "str",
84
+ }
85
+ TEST_INPUT_FILE = {
86
+ PMAP_SHORT: "-TT",
87
+ PMAP_LONG: "--test-file",
88
+ PMAP_DEST: "test_file",
89
+ PMAP_DESCR: "Path to test input file",
90
+ PMAP_TYPE: "str",
91
+ }
92
+ VALID_INPUT_FILE = {
93
+ PMAP_SHORT: "-V",
94
+ PMAP_LONG: "--valid-file",
95
+ PMAP_DEST: "valid_file",
96
+ PMAP_DESCR: "Path to train input file",
97
+ PMAP_TYPE: "str",
98
+ }
99
+ BASE_MODEL = {
100
+ PMAP_SHORT: "-m",
101
+ PMAP_LONG: "--base-model",
102
+ PMAP_DEST: "base_model",
103
+ PMAP_DESCR: "Name or path to base model to be used.",
104
+ PMAP_TYPE: "str",
105
+ }
106
+ REWARD_MODEL = {
107
+ PMAP_SHORT: "-rlrm",
108
+ PMAP_LONG: "--reward-model",
109
+ PMAP_DEST: "reward_model",
110
+ PMAP_DESCR: "Name or path to reward model to be used in case of reinforcement learning.",
111
+ PMAP_TYPE: "str",
112
+ }
113
+ WANDB_ONOFF = {
114
+ PMAP_SHORT: "-WB",
115
+ PMAP_LONG: "--use-wandb-full",
116
+ PMAP_DEST: "wandb_full",
117
+ PMAP_DESCR: "If enabled then full W&B integration will be enabled. Full "
118
+ "logging, dataset as artifact storing and models storing",
119
+ PMAP_TYPE: "bool",
120
+ }
121
+
122
+ INPUT_FILE_REQUIRED = APMD.INPUT_FILE.copy()
123
+ INPUT_FILE_REQUIRED[APMD.PMAP_REQ] = True
124
+ INPUT_FILE_NOT_REQUIRED = APMD.INPUT_FILE.copy()
125
+ INPUT_FILE_NOT_REQUIRED[APMD.PMAP_REQ] = False
126
+
127
+ INPUT_DIR_REQUIRED = APMD.INPUT_DIR.copy()
128
+ INPUT_DIR_REQUIRED[APMD.PMAP_REQ] = True
129
+ INPUT_DIR_NOT_REQUIRED = APMD.INPUT_FILE.copy()
130
+ INPUT_DIR_NOT_REQUIRED[APMD.PMAP_REQ] = False
131
+
132
+ OUTPUT_FILE_REQUIRED = APMD.OUTPUT_FILE.copy()
133
+ OUTPUT_FILE_REQUIRED[APMD.PMAP_REQ] = True
134
+ OUTPUT_FILE_NOT_REQUIRED = APMD.OUTPUT_FILE.copy()
135
+ OUTPUT_FILE_NOT_REQUIRED[APMD.PMAP_REQ] = False
136
+
137
+ OUTPUT_DIR_REQUIRED = APMD.OUTPUT_DIR.copy()
138
+ OUTPUT_DIR_REQUIRED[APMD.PMAP_REQ] = True
139
+ OUTPUT_DIR_NOT_REQUIRED = APMD.OUTPUT_DIR.copy()
140
+ OUTPUT_DIR_NOT_REQUIRED[APMD.PMAP_REQ] = False
141
+
142
+ WORK_DIR_REQUIRED = APMD.WORK_DIR.copy()
143
+ WORK_DIR_REQUIRED[APMD.PMAP_REQ] = True
144
+ WORK_DIR_NOT_REQUIRED = APMD.WORK_DIR.copy()
145
+ WORK_DIR_NOT_REQUIRED[APMD.PMAP_REQ] = False
146
+
147
+ BASE_MODEL_REQUIRED = APMD.BASE_MODEL.copy()
148
+ BASE_MODEL_REQUIRED[APMD.PMAP_REQ] = True
149
+ BASE_MODEL_NOT_REQUIRED = APMD.BASE_MODEL.copy()
150
+ BASE_MODEL_NOT_REQUIRED[APMD.PMAP_REQ] = False
151
+
152
+ REWARD_MODEL_REQUIRED = APMD.REWARD_MODEL.copy()
153
+ REWARD_MODEL_REQUIRED[APMD.PMAP_REQ] = True
154
+ REWARD_MODEL_NOT_REQUIRED = APMD.REWARD_MODEL.copy()
155
+ REWARD_MODEL_NOT_REQUIRED[APMD.PMAP_REQ] = False
156
+
157
+ TRAIN_FILE_REQUIRED = APMD.TRAIN_INPUT_FILE.copy()
158
+ TRAIN_FILE_REQUIRED[APMD.PMAP_REQ] = True
159
+ TRAIN_FILE_NOT_REQUIRED = APMD.TRAIN_INPUT_FILE.copy()
160
+ TRAIN_FILE_NOT_REQUIRED[APMD.PMAP_REQ] = False
161
+
162
+ TEST_FILE_REQUIRED = APMD.TEST_INPUT_FILE.copy()
163
+ TEST_FILE_REQUIRED[APMD.PMAP_REQ] = True
164
+ TEST_FILE_NOT_REQUIRED = APMD.TEST_INPUT_FILE.copy()
165
+ TEST_FILE_NOT_REQUIRED[APMD.PMAP_REQ] = False
166
+
167
+ VALID_FILE_REQUIRED = APMD.VALID_INPUT_FILE.copy()
168
+ VALID_FILE_REQUIRED[APMD.PMAP_REQ] = True
169
+ VALID_FILE_NOT_REQUIRED = APMD.VALID_INPUT_FILE.copy()
170
+ VALID_FILE_NOT_REQUIRED[APMD.PMAP_REQ] = False
171
+
172
+ WANDB_BOOL_ONOFF = APMD.WANDB_ONOFF.copy()
173
+
174
+
175
+ # There are mappings param name to argument parser item
176
+ PARSER_FIELDS_MAP = {
177
+ INPUT_FILE_REQUIRED: _APM.INPUT_FILE_REQUIRED,
178
+ INPUT_FILE_NOT_REQUIRED: _APM.INPUT_FILE_NOT_REQUIRED,
179
+ INPUT_DIR_REQUIRED: _APM.INPUT_DIR_REQUIRED,
180
+ INPUT_DIR_NOT_REQUIRED: _APM.INPUT_DIR_NOT_REQUIRED,
181
+ OUTPUT_FILE_REQUIRED: _APM.OUTPUT_FILE_REQUIRED,
182
+ OUTPUT_FILE_NOT_REQUIRED: _APM.OUTPUT_FILE_NOT_REQUIRED,
183
+ OUTPUT_DIR_REQUIRED: _APM.OUTPUT_DIR_REQUIRED,
184
+ OUTPUT_DIR_NOT_REQUIRED: _APM.OUTPUT_DIR_NOT_REQUIRED,
185
+ WORK_DIR_REQUIRED: _APM.WORK_DIR_REQUIRED,
186
+ WORK_DIR_NOT_REQUIRED: _APM.WORK_DIR_NOT_REQUIRED,
187
+ BASE_MODEL_REQUIRED: _APM.BASE_MODEL_REQUIRED,
188
+ BASE_MODEL_NOT_REQUIRED: _APM.BASE_MODEL_NOT_REQUIRED,
189
+ REWARD_MODEL_REQUIRED: _APM.REWARD_MODEL_REQUIRED,
190
+ REWARD_MODEL_NOT_REQUIRED: _APM.REWARD_MODEL_NOT_REQUIRED,
191
+ TRAIN_INPUT_FILE_REQUIRED: _APM.TRAIN_FILE_REQUIRED,
192
+ TRAIN_INPUT_FILE_NOT_REQUIRED: _APM.TRAIN_FILE_NOT_REQUIRED,
193
+ TEST_INPUT_FILE_REQUIRED: _APM.TEST_FILE_REQUIRED,
194
+ TEST_INPUT_FILE_NOT_REQUIRED: _APM.TEST_FILE_NOT_REQUIRED,
195
+ VALID_INPUT_FILE_REQUIRED: _APM.VALID_FILE_REQUIRED,
196
+ VALID_INPUT_FILE_NOT_REQUIRED: _APM.VALID_FILE_NOT_REQUIRED,
197
+ WANDB_BOOLEAN_FULL: _APM.WANDB_BOOL_ONOFF,
198
+ }
199
+
200
+
201
+ def base_argument_parser(desc=""):
202
+ p = ArgumentParser(description=desc)
203
+ return p
204
+
205
+
206
+ def prepare_parser_for_fields(
207
+ fields_list: list, description: str = None
208
+ ) -> ArgumentParser:
209
+ """
210
+ It's a general utility function which may be used to prepare ArgumentParser
211
+ with predefined options.
212
+
213
+ :param fields_list: List of fields to construct the parser
214
+ :param description: Application/parser description
215
+ :return: Object of ArgumentParser
216
+ """
217
+ p = base_argument_parser(description if description is not None else "")
218
+
219
+ for item in fields_list:
220
+ if item not in PARSER_FIELDS_MAP:
221
+ raise Exception(f"Cannot find argument parser mapping for item: {item} ")
222
+
223
+ p_item = PARSER_FIELDS_MAP[item]
224
+
225
+ if p_item[_APM.APMD.PMAP_TYPE] in ["bool"]:
226
+ p.add_argument(
227
+ p_item[_APM.APMD.PMAP_SHORT],
228
+ p_item[_APM.APMD.PMAP_LONG],
229
+ help=p_item[_APM.APMD.PMAP_DESCR],
230
+ dest=p_item[_APM.APMD.PMAP_DEST],
231
+ action="store_true",
232
+ )
233
+ else:
234
+ p.add_argument(
235
+ p_item[_APM.APMD.PMAP_SHORT],
236
+ p_item[_APM.APMD.PMAP_LONG],
237
+ help=p_item[_APM.APMD.PMAP_DESCR],
238
+ dest=p_item[_APM.APMD.PMAP_DEST],
239
+ required=p_item[_APM.APMD.PMAP_REQ],
240
+ )
241
+ return p
@@ -0,0 +1,95 @@
1
+ import json
2
+
3
+ from transformers import HfArgumentParser, TrainingArguments
4
+
5
+
6
+ class HFTrainingArgumentHandler:
7
+ def __init__(
8
+ self,
9
+ out_dir: str = None,
10
+ report_to: list = None,
11
+ tr_arguments: TrainingArguments = None,
12
+ ):
13
+ """
14
+ FT TrainingArgument constructor
15
+ :param out_dir:
16
+ :param report_to:
17
+ :param tr_arguments:
18
+ """
19
+ if tr_arguments is not None:
20
+ self._tr_arguments = tr_arguments
21
+ else:
22
+ self._tr_arguments = self._get_default_training_arguments(
23
+ out_dir=out_dir, report_to=report_to
24
+ )
25
+
26
+ def to_dict(self):
27
+ """
28
+ Get dict with training_arguments.
29
+ """
30
+ data = {"training_arguments": self.training_arguments.to_dict()}
31
+ return data
32
+
33
+ def save(self, out_filepath: str):
34
+ """
35
+ Store training arguments as json file
36
+ :param out_filepath:
37
+ :return:
38
+ """
39
+ data = self.to_dict()
40
+ with open(out_filepath, "w") as file:
41
+ json.dump(data, file, indent=4)
42
+
43
+ @classmethod
44
+ def from_config(cls, config_json: str):
45
+ """
46
+ Loads training arguments from json config
47
+ :param config_json:
48
+ :return: object of HFTrainingArgumentHandler
49
+ """
50
+ with open(config_json, mode="r") as file:
51
+ data = json.load(file)
52
+ training_args_dict = data["training_arguments"]
53
+ parser = HfArgumentParser(TrainingArguments)
54
+ (training_args,) = parser.parse_dict(training_args_dict)
55
+ return cls(tr_arguments=training_args)
56
+
57
+ @property
58
+ def training_arguments(self):
59
+ return self._tr_arguments
60
+
61
+ def _get_default_training_arguments(
62
+ self, out_dir: str = None, report_to: list = None
63
+ ):
64
+ """
65
+ Default training arguments initialization.
66
+ :param out_dir:
67
+ :param report_to:
68
+ :return: object of TrainingArguments
69
+ """
70
+ ta = TrainingArguments(
71
+ output_dir=out_dir,
72
+ num_train_epochs=50,
73
+ per_device_train_batch_size=16,
74
+ per_device_eval_batch_size=16,
75
+ warmup_steps=200,
76
+ weight_decay=0.001,
77
+ load_best_model_at_end=True,
78
+ logging_steps=10,
79
+ eval_strategy="steps",
80
+ eval_steps=40,
81
+ disable_tqdm=False,
82
+ logging_first_step=False,
83
+ learning_rate=0.0005,
84
+ fp16=True,
85
+ report_to=report_to,
86
+ save_total_limit=5,
87
+ save_strategy="steps",
88
+ save_steps=40000,
89
+ logging_strategy="steps",
90
+ gradient_accumulation_steps=1,
91
+ eval_accumulation_steps=1,
92
+ optim="adamw_torch_fused",
93
+ fp16_full_eval=True,
94
+ )
95
+ return ta
@@ -0,0 +1,43 @@
1
+ import torch
2
+ from sklearn.metrics import (
3
+ f1_score,
4
+ accuracy_score,
5
+ precision_score,
6
+ recall_score,
7
+ )
8
+
9
+
10
+ def preds_to_class_label_softmax(predictions):
11
+ sft_max = torch.nn.Softmax(dim=0)
12
+ sft_preds = sft_max(torch.as_tensor(predictions, dtype=torch.float))
13
+ return sft_preds
14
+
15
+
16
+ def preds_to_class_label_argmax(predictions):
17
+ preds = [torch.argmax(p) for p in predictions]
18
+ return preds
19
+
20
+
21
+ def compute_metrics(preds, use_softmax=True, use_argmax=True):
22
+ preds_cl = preds
23
+
24
+ if use_softmax:
25
+ preds_cl = preds_to_class_label_softmax(predictions=preds.predictions)
26
+
27
+ if use_argmax:
28
+ preds_cl = preds_to_class_label_argmax(predictions=preds_cl)
29
+
30
+ prec_val = precision_score(y_true=preds.label_ids, y_pred=preds_cl, average="macro")
31
+ recall_val = recall_score(y_true=preds.label_ids, y_pred=preds_cl, average="macro")
32
+ f1_val_macro = f1_score(y_true=preds.label_ids, y_pred=preds_cl, average="macro")
33
+ # conf_m_val = confusion_matrix(
34
+ # y_true=preds.label_ids, y_pred=preds_cl
35
+ # )
36
+ acc_val = accuracy_score(y_true=preds.label_ids, y_pred=preds_cl)
37
+ return {
38
+ "f1": f1_val_macro,
39
+ "precision": prec_val,
40
+ "recall": recall_val,
41
+ "accuracy": acc_val,
42
+ # 'confusion_matrix': conf_m_val
43
+ }