radlab-data 0.0.4__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. radlab_data/__init__.py +9 -0
  2. radlab_data/datasets/__init__.py +0 -0
  3. radlab_data/datasets/dataset.py +509 -0
  4. radlab_data/datasets/dataset_loader.py +140 -0
  5. radlab_data/datasets/tokenization_handler.py +78 -0
  6. radlab_data/preprocessing/__init__.py +0 -0
  7. radlab_data/preprocessing/dataset.py +726 -0
  8. radlab_data/preprocessing/pipeline.py +38 -0
  9. radlab_data/preprocessing/pipeline_modules.py +342 -0
  10. radlab_data/text/__init__.py +0 -0
  11. radlab_data/text/document.py +136 -0
  12. radlab_data/text/loaders/__init__.py +0 -0
  13. radlab_data/text/loaders/base.py +65 -0
  14. radlab_data/text/loaders/config.py +4 -0
  15. radlab_data/text/loaders/docx.py +96 -0
  16. radlab_data/text/loaders/html.py +111 -0
  17. radlab_data/text/loaders/input_txt.py +30 -0
  18. radlab_data/text/loaders/loader_i.py +267 -0
  19. radlab_data/text/loaders/markdown.py +129 -0
  20. radlab_data/text/loaders/pdf.py +157 -0
  21. radlab_data/text/loaders/txt.py +44 -0
  22. radlab_data/text/pair_reader.py +241 -0
  23. radlab_data/text/processors/__init__.py +0 -0
  24. radlab_data/text/processors/base.py +23 -0
  25. radlab_data/text/processors/cleaner.py +338 -0
  26. radlab_data/text/processors/mappers.py +167 -0
  27. radlab_data/text/processors/regex_defs.py +58 -0
  28. radlab_data/text/processors/splitter.py +60 -0
  29. radlab_data/text/reader.py +198 -0
  30. radlab_data/text/utils.py +51 -0
  31. radlab_data/utils/__init__.py +0 -0
  32. radlab_data/utils/api_handler.py +83 -0
  33. radlab_data/utils/argument_parser.py +241 -0
  34. radlab_data/utils/hf_training_arguments.py +95 -0
  35. radlab_data/utils/metrics.py +43 -0
  36. radlab_data/utils/special_tokens_rules.py +330 -0
  37. radlab_data/utils/threads.py +61 -0
  38. radlab_data-0.0.4.dist-info/METADATA +290 -0
  39. radlab_data-0.0.4.dist-info/RECORD +42 -0
  40. radlab_data-0.0.4.dist-info/WHEEL +5 -0
  41. radlab_data-0.0.4.dist-info/licenses/LICENSE +201 -0
  42. radlab_data-0.0.4.dist-info/top_level.txt +1 -0
@@ -0,0 +1,330 @@
1
+ from collections import defaultdict
2
+ from typing import List
3
+
4
+ import tqdm
5
+
6
+ OPERATORS = {
7
+ ">": {"func": lambda x, y: x > y, "priority": 0},
8
+ "<": {"func": lambda x, y: x < y, "priority": 0},
9
+ ">=": {"func": lambda x, y: x >= y, "priority": 0},
10
+ "<=": {"func": lambda x, y: x <= y, "priority": 0},
11
+ "==": {"func": lambda x, y: x == y, "priority": 0},
12
+ "and": {"func": lambda x, y: x and y, "priority": 1},
13
+ "or": {"func": lambda x, y: x or y, "priority": 2},
14
+ }
15
+
16
+
17
+ class ApplicatorNode:
18
+ NODE_TYPES = ["operator", "accessor", "value"]
19
+
20
+ def __init__(self, type) -> None:
21
+ assert type in self.NODE_TYPES
22
+ self.type = type
23
+
24
+ self.value = None
25
+ self.child_a: ApplicatorNode = None
26
+ self.child_b: ApplicatorNode = None
27
+ self.operator_method = None
28
+ self.access_method = None
29
+ self.str_representation = None
30
+
31
+ def _evaluate(self, data):
32
+ if self.type == "accessor":
33
+ return self._access(data)
34
+ elif self.type == "operator":
35
+ return self._operation(data)
36
+ elif self.type == "value":
37
+ return self._get_value()
38
+
39
+ def _access(self, data):
40
+ return self.access_method(data)
41
+
42
+ def _operation(self, data):
43
+ return self.operator_method(
44
+ self.child_a._evaluate(data), self.child_b._evaluate(data)
45
+ )
46
+
47
+ def _get_value(self):
48
+ return self.value
49
+
50
+ def __str__(self) -> str:
51
+ return f"{self.type} [{self.str_representation}]"
52
+
53
+
54
+ class PlacementAccessor:
55
+ WHERE_KEYS = ["beginning", "end"]
56
+
57
+ def __init__(self, object: dict, class_label_field_name: str = "labels") -> None:
58
+ self.class_label_field_name = class_label_field_name
59
+ self.object = object
60
+
61
+ def set_value(self, placement_str: str, value):
62
+ access_keys = placement_str.split("::")
63
+ where = access_keys[-1]
64
+ assert len(access_keys) > 1
65
+ assert where in self.WHERE_KEYS
66
+ first_key = access_keys[0]
67
+ object_ptr = self.object
68
+ accessed_data = []
69
+ if first_key == "text":
70
+ assert access_keys[1] == where
71
+ accessed_data.append((object_ptr, "text"))
72
+ elif first_key == self.class_label_field_name:
73
+ assert access_keys[1] == where
74
+ object_ptr = object_ptr[self.class_label_field_name]
75
+ for idx in range(len(object_ptr)):
76
+ accessed_data.append((object_ptr, idx))
77
+ else:
78
+ object_ptr = object_ptr["metadata"]
79
+ for key in access_keys[:-2]:
80
+ object_ptr = object_ptr[key]
81
+ last_ptr = object_ptr[access_keys[-2]]
82
+ if isinstance(last_ptr, list):
83
+ object_ptr = last_ptr
84
+ for idx in range(len(object_ptr)):
85
+ accessed_data.append((object_ptr, idx))
86
+ else:
87
+ accessed_data.append((object_ptr, access_keys[-2]))
88
+
89
+ for ptr, key in accessed_data:
90
+ ptr[key] = self._modify_string(ptr[key], value, where)
91
+
92
+ @staticmethod
93
+ def _modify_string(string: str, value: str, where: str):
94
+ if where == "beginning":
95
+ return value + string
96
+ elif where == "end":
97
+ return string + value
98
+
99
+
100
+ class ApplicatorTree:
101
+ def __init__(
102
+ self,
103
+ root: ApplicatorNode,
104
+ rule_str: str,
105
+ placement: str,
106
+ special_token: str,
107
+ ) -> None:
108
+ self.root: ApplicatorNode = root
109
+ if self.root.type != "operator":
110
+ raise ValueError(f"The following rule contains no operators: {rule_str}")
111
+ self.placement: str = placement
112
+ self.special_token: str = special_token
113
+
114
+ def _evaluate(self, data):
115
+ return self.root._evaluate(data)
116
+
117
+ def print_tree(self):
118
+ print(f"{self.__class__.__name__}")
119
+ print(str(self))
120
+
121
+ def _get_str_representation(self):
122
+ return self._get_str_representation_recursive(self.root, 0)
123
+
124
+ def _get_str_representation_recursive(self, node, depth):
125
+ if node is not None:
126
+ string = "\t" * depth + str(node)
127
+ if node.child_a is not None:
128
+ string += "\n" + self._get_str_representation_recursive(
129
+ node.child_a, depth + 1
130
+ )
131
+ if node.child_b is not None:
132
+ string += "\n" + self._get_str_representation_recursive(
133
+ node.child_b, depth + 1
134
+ )
135
+ return string
136
+ return ""
137
+
138
+ def __str__(self) -> str:
139
+ return self._get_str_representation()
140
+
141
+ def __call__(self, *args, **kwargs):
142
+ return self._evaluate(*args, **kwargs)
143
+
144
+
145
+ class RuleApplicator:
146
+ def __init__(
147
+ self, rule_trees: List[ApplicatorTree], special_tokens: List[str]
148
+ ) -> None:
149
+ self.rule_trees: List[ApplicatorTree] = rule_trees
150
+ self.special_tokens: List[str] = special_tokens
151
+
152
+ def _process(self, data):
153
+ for elem in tqdm.tqdm(data):
154
+ true_trees = [tree for tree in self.rule_trees if tree(elem)]
155
+ self._apply_trees(true_trees, elem)
156
+
157
+ @staticmethod
158
+ def _apply_trees(trees: List[ApplicatorTree], elem: dict):
159
+ accessor = PlacementAccessor(elem)
160
+ placement_buckets = defaultdict(list)
161
+ all_placements = [tree.placement for tree in trees]
162
+ all_special_tokens = [tree.special_token for tree in trees]
163
+ for placement, special_token in zip(all_placements, all_special_tokens):
164
+ placement_buckets[placement].append(special_token)
165
+ for placement, bucket_values in placement_buckets.items():
166
+ tokens_fused = "".join(bucket_values)
167
+ accessor.set_value(placement, tokens_fused)
168
+
169
+ def __call__(self, data):
170
+ return self._process(data)
171
+
172
+
173
+ class SpecialTokensRulesParser:
174
+ def __init__(self, special_tokens) -> None:
175
+ self.special_tokens = special_tokens
176
+
177
+ def _merge_rules(self, special_token_rules):
178
+ groups = defaultdict(list)
179
+ for rule in special_token_rules:
180
+ group_unique_id = rule["token"] + " " + rule["placement"]
181
+ groups[group_unique_id].append(rule)
182
+ new_rules = []
183
+ for group in groups.values():
184
+ if len(group) == 1:
185
+ rule = group[0]
186
+ else:
187
+ when = " or ".join(f"({s['when']})" for s in group)
188
+ rule = {
189
+ "token": group[0]["token"],
190
+ "placement": group[0]["placement"],
191
+ "when": when,
192
+ }
193
+ new_rules.append(rule)
194
+ return new_rules
195
+
196
+ def _sort_rules(self, special_tokens_rules):
197
+ buckets = {k: [] for k in self.special_tokens} # ordered dict (Python 3.7)
198
+ for rule in special_tokens_rules:
199
+ token = rule["token"]
200
+ if token not in buckets:
201
+ raise KeyError(
202
+ f"Special token {token} used in the rule "
203
+ f"{rule['when']!r} is not in the list of declared "
204
+ f"special tokens: {self.special_tokens}."
205
+ )
206
+ buckets[token].append(rule)
207
+ new_rules = []
208
+ for bucket in buckets.values():
209
+ new_rules.extend(bucket)
210
+ return new_rules
211
+
212
+ def parse(self, special_tokens_rules) -> List[ApplicatorTree]:
213
+ special_tokens_rules = self._merge_rules(special_tokens_rules)
214
+ special_tokens_rules = self._sort_rules(special_tokens_rules)
215
+
216
+ rule_trees: List[ApplicatorTree] = []
217
+ for rule in special_tokens_rules:
218
+ token = rule["token"]
219
+ when = rule["when"]
220
+ placement = rule["placement"]
221
+
222
+ when_words = list(
223
+ enumerate(when.replace("(", " ( ").replace(")", " ) ").split())
224
+ ) # whitespace split
225
+
226
+ root_node = self.parse_when_words(when_words, when)
227
+ when_tree = ApplicatorTree(
228
+ root=root_node,
229
+ rule_str=when,
230
+ placement=placement,
231
+ special_token=token,
232
+ )
233
+ rule_trees.append(when_tree)
234
+ return rule_trees
235
+
236
+ def parse_when_words(self, when_words, sentence):
237
+ try:
238
+ return self._parse_when_words_recursive(when_words)
239
+ except (AssertionError, KeyError, IndexError, ValueError) as error:
240
+ raise Exception(
241
+ f"Error has occured while parsing the following rule: {sentence}"
242
+ ) from error
243
+
244
+ def _are_outer_brackets_present(self, expression):
245
+ if (
246
+ len(expression) >= 3
247
+ and expression[0][1] == "("
248
+ and expression[-1][1] == ")"
249
+ ):
250
+ stack = 0
251
+ for _, word in expression[1:-1]:
252
+ if word == "(":
253
+ stack += 1
254
+ elif word == ")":
255
+ if stack == 0:
256
+ return False
257
+ stack -= 1
258
+ return stack == 0
259
+ return False
260
+
261
+ def _parse_when_words_recursive(
262
+ self, when_words: list, used_operators: list = None
263
+ ) -> ApplicatorNode:
264
+ if self._are_outer_brackets_present(when_words):
265
+ when_words = when_words[1:-1]
266
+
267
+ used_operators = []
268
+ bracket_count = 0
269
+ for idx, word in when_words:
270
+ if word == "(":
271
+ bracket_count += 1
272
+ elif word == ")":
273
+ bracket_count -= 1
274
+ elif word in OPERATORS and bracket_count == 0:
275
+ operator = OPERATORS[word]
276
+ used_operators.append(
277
+ {"word_idx": idx, "operator": operator, "word": word}
278
+ )
279
+
280
+ if len(used_operators) == 0:
281
+ assert len(when_words) == 1
282
+ word = when_words[0][
283
+ 1
284
+ ] # 0 because first word in the list, and 1 because index 0 is the index of a word and index 1 is the string
285
+
286
+ is_value = word.replace(".", "").isnumeric()
287
+ if is_value:
288
+ node = ApplicatorNode(type="value")
289
+ node.value = float(word)
290
+ node.str_representation = word
291
+ return node
292
+
293
+ access_keys = word.split("::")
294
+
295
+ def access_method(x):
296
+ first_key = access_keys[0]
297
+ if first_key not in ["text", "labels"]:
298
+ x = x["metadata"]
299
+ for access_key in access_keys:
300
+ x = x[access_key]
301
+ return x
302
+
303
+ node = ApplicatorNode(type="accessor")
304
+ node.access_method = access_method
305
+ node.str_representation = word
306
+ return node
307
+ else:
308
+ active_operator = max(
309
+ used_operators,
310
+ key=lambda x: (x["operator"]["priority"], x["word_idx"]),
311
+ )
312
+ split_point = active_operator["word_idx"]
313
+ words_left = [w for w in when_words if w[0] < split_point]
314
+ words_right = [w for w in when_words if w[0] > split_point]
315
+ assert len(words_left) > 0
316
+ assert len(words_right) > 0
317
+ operators_left = [
318
+ op for op in used_operators if op["word_idx"] < split_point
319
+ ]
320
+ operators_right = [
321
+ op for op in used_operators if op["word_idx"] > split_point
322
+ ]
323
+ node = ApplicatorNode(type="operator")
324
+ node.child_a = self._parse_when_words_recursive(words_left, operators_left)
325
+ node.child_b = self._parse_when_words_recursive(
326
+ words_right, operators_right
327
+ )
328
+ node.operator_method = active_operator["operator"]["func"]
329
+ node.str_representation = f"{active_operator['word']}"
330
+ return node
@@ -0,0 +1,61 @@
1
+ import threading
2
+
3
+ from typing import List
4
+ from abc import ABC, abstractmethod
5
+
6
+
7
+ class ThreadWorker(ABC):
8
+ def __init__(self, debug: bool = False) -> None:
9
+ self._debug = debug
10
+ self._thread = threading.Thread(target=self._task)
11
+ self._enabled = True
12
+
13
+ def start(self) -> None:
14
+ if not self._enabled:
15
+ raise Exception("Cannot run again a disabled task.")
16
+ if self._debug:
17
+ print(f"Task running {self.__class__.__name__}")
18
+ self._thread.start()
19
+
20
+ @abstractmethod
21
+ def _task(self) -> None:
22
+ pass
23
+
24
+ def disable(self):
25
+ if self._debug:
26
+ print(f"Task disabled {self.__class__.__name__}")
27
+ self._enabled = False
28
+
29
+ def join(self):
30
+ self._thread.join()
31
+ if self._debug:
32
+ print(f"Task finished {self.__class__.__name__}")
33
+
34
+
35
+ class WorkerCluster:
36
+ def __init__(
37
+ self, name: str, threads: List[ThreadWorker], debug: bool = False
38
+ ) -> None:
39
+ self._debug = debug
40
+ self._name = self.__class__.__name__
41
+ if name is not None:
42
+ self._name = name
43
+ self._threads = threads
44
+
45
+ def start(self):
46
+ if self._debug:
47
+ print(f"Running cluster {self._name}")
48
+ for thread in self._threads:
49
+ thread.start()
50
+
51
+ def disable(self):
52
+ if self._debug:
53
+ print(f"Disabling cluster {self._name}")
54
+ for thread in self._threads:
55
+ thread.disable()
56
+
57
+ def join(self):
58
+ if self._debug:
59
+ print(f"Joining cluster {self._name}")
60
+ for thread in self._threads:
61
+ thread.join()
@@ -0,0 +1,290 @@
1
+ Metadata-Version: 2.4
2
+ Name: radlab-data
3
+ Version: 0.0.4
4
+ Summary: Text processing and modelling
5
+ Author-email: RadLab <hello@radlab.dev>
6
+ License: Apache-2.0
7
+ Project-URL: Homepage, https://radlab.dev
8
+ Classifier: Development Status :: 4 - Beta
9
+ Classifier: Intended Audience :: Developers
10
+ Classifier: Intended Audience :: Science/Research
11
+ Classifier: License :: OSI Approved :: Apache Software License
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3.10
15
+ Classifier: Programming Language :: Python :: 3.11
16
+ Classifier: Programming Language :: Python :: 3.12
17
+ Classifier: Programming Language :: Python :: 3.13
18
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
19
+ Classifier: Topic :: Text Processing
20
+ Requires-Python: >=3.10
21
+ Description-Content-Type: text/markdown
22
+ License-File: LICENSE
23
+ Requires-Dist: tqdm
24
+ Requires-Dist: requests
25
+ Requires-Dist: beautifulsoup4
26
+ Requires-Dist: python-docx
27
+ Requires-Dist: pypdf
28
+ Provides-Extra: ml
29
+ Requires-Dist: transformers; extra == "ml"
30
+ Requires-Dist: torch; extra == "ml"
31
+ Requires-Dist: datasets; extra == "ml"
32
+ Requires-Dist: numpy; extra == "ml"
33
+ Requires-Dist: scikit-learn; extra == "ml"
34
+ Requires-Dist: scipy; extra == "ml"
35
+ Requires-Dist: matplotlib; extra == "ml"
36
+ Provides-Extra: spacy
37
+ Requires-Dist: spacy; extra == "spacy"
38
+ Provides-Extra: lang
39
+ Requires-Dist: fasttext-langdetect; extra == "lang"
40
+ Requires-Dist: deepl; extra == "lang"
41
+ Provides-Extra: tables
42
+ Requires-Dist: pandas; extra == "tables"
43
+ Requires-Dist: openpyxl; extra == "tables"
44
+ Provides-Extra: all
45
+ Requires-Dist: transformers; extra == "all"
46
+ Requires-Dist: torch; extra == "all"
47
+ Requires-Dist: datasets; extra == "all"
48
+ Requires-Dist: numpy; extra == "all"
49
+ Requires-Dist: scikit-learn; extra == "all"
50
+ Requires-Dist: scipy; extra == "all"
51
+ Requires-Dist: matplotlib; extra == "all"
52
+ Requires-Dist: spacy; extra == "all"
53
+ Requires-Dist: fasttext-langdetect; extra == "all"
54
+ Requires-Dist: deepl; extra == "all"
55
+ Requires-Dist: pandas; extra == "all"
56
+ Requires-Dist: openpyxl; extra == "all"
57
+ Dynamic: license-file
58
+
59
+ # radlab‑data
60
+
61
+ A versatile Python library for loading, preprocessing, and handling a variety of textual data sources (PDF, DOCX, TXT, MD, HTML,
62
+ and raw strings) and preparing them for downstream machine‑learning tasks such as classification, sequence labeling, and
63
+ generative modeling.
64
+
65
+ ---
66
+
67
+ ## Table of Contents
68
+
69
+ - [Overview](#overview)
70
+ - [Features](#features)
71
+ - [Installation](#installation)
72
+ - [Quick Start](#quick-start)
73
+ - [Core Concepts](#core-concepts)
74
+ - [Dataset Types](#dataset-types)
75
+ - [Text Loaders](#text-loaders)
76
+ - [Processing Pipeline](#processing-pipeline)
77
+ - [Command‑Line Utilities](#command-line-utilities)
78
+ - [Configuration](#configuration)
79
+ - [License](#license)
80
+
81
+ ---
82
+
83
+ ## Overview
84
+
85
+ `radlab-data` provides a unified interface for:
86
+
87
+ 1. **Reading documents** from common file formats (PDF, DOCX, TXT, MD, HTML) as well as plain‑text strings.
88
+ 2. **Cleaning and normalising** raw text using a chain of configurable processors.
89
+ 3. **Splitting** text into sentences, paragraphs, or token‑level chunks.
90
+ 4. **Mapping** raw annotations (e.g., NER spans) onto tokenised representations.
91
+ 5. **Packaging** data into HuggingFace `datasets.Dataset` objects, ready for training with `transformers`.
92
+
93
+ The library is built with extensibility in mind – you can plug in your own tokenizers, preprocessing steps, or custom
94
+ dataset classes.
95
+
96
+ ---
97
+
98
+ ## Features
99
+
100
+ - **Multi‑format document loading** – PDF, DOCX, TXT, MD, HTML, and custom “input_text”.
101
+ - **Automatic language detection** (via `ftlangdetect`).
102
+ - **Flexible text cleaning** – phrase removal, abbreviation normalisation, regex‑based sanitisation, etc.
103
+ - **Sentence‑level splitting** using spaCy’s Polish model (`pl_core_news_lg`).
104
+ - **Special token handling** – conditional insertion of tokens based on metadata rules.
105
+ - **Dataset abstractions** for:
106
+ - Text classification
107
+ - Sequence labeling (NER, IOB tagging)
108
+ - Sequence‑to‑sequence tasks (e.g., translation, summarisation)
109
+ - Semantic similarity and dialogue alternatives
110
+ - **Caching** of tokenised datasets for fast reloads.
111
+ - **Multiprocessing support** for loading large corpora.
112
+ - **Command‑line tools** for dataset conversion, preprocessing, and training argument handling.
113
+
114
+ ---
115
+
116
+ ## Installation
117
+
118
+ ```shell script
119
+ # Create a virtual environment (optional but recommended)
120
+ python -m venv .venv
121
+ source .venv/bin/activate # on Windows: .venv\Scripts\activate
122
+
123
+ # Core only: document loading, cleaning and small utilities
124
+ pip install -e .
125
+
126
+ # Everything, including the model stack
127
+ pip install -e ".[all]"
128
+ ```
129
+
130
+ > **Note**: The library requires Python 3.10+.
131
+
132
+ ### Dependency extras
133
+
134
+ The core installs only the light packages needed to read and clean documents
135
+ (`tqdm`, `requests`, `beautifulsoup4`, `python-docx`, `pypdf`). Importing the
136
+ document loaders, `Document`, `DirectoryFileReader` or `TextUtils` therefore
137
+ does **not** pull in `torch`, `transformers` or `spacy` — a query-only host
138
+ (for example the [Semantic Search Engine](https://dev.radlab.pl/semantic-search-engine) API)
139
+ can install the core alone.
140
+
141
+ | Extra | Installs | Needed for |
142
+ |------------|-------------------------------------------------------------------|------------------------------------------------------------------|
143
+ | `ml` | `transformers`, `torch`, `datasets`, `numpy`, `scikit-learn`, `scipy`, `matplotlib` | token-level chunking (`max_tokens_in_chunk`), HF datasets, metrics and training helpers |
144
+ | `spacy` | `spacy` | `SentenceSplitter` and `PairReader` (Polish model: `python -m spacy download pl_core_news_lg`) |
145
+ | `lang` | `fasttext-langdetect`, `deepl` | `TextUtils.text_language`, `TextUtils.translate_text_deepl` |
146
+ | `tables` | `pandas`, `openpyxl` | dataset preprocessing and spreadsheet input |
147
+ | `all` | all of the above | the full feature set |
148
+
149
+ Heavy imports are lazy: `transformers` is imported inside
150
+ `radlab_data.text.loaders.loader_i.get_tokenizer()` (so inside the first use of
151
+ token chunking), `spacy` inside
152
+ `radlab_data.text.processors.splitter.get_nlp()`, and `ftlangdetect`/`deepl`
153
+ inside the `TextUtils` methods which use them. `requirements.txt` lists every
154
+ extra for a full development environment.
155
+
156
+ ---
157
+
158
+ ## Quick Start
159
+
160
+ ```python
161
+ from radlab_data.text.document import Document
162
+ from radlab_data.datasets.dataset_loader import DatasetLoader
163
+ from transformers import AutoTokenizer
164
+
165
+ # 1️⃣ Load a document (PDF, DOCX, TXT, MD, HTML, or raw text)
166
+ doc = Document(
167
+ file_path="data/example.pdf",
168
+ prepare_proper_pages=True,
169
+ clear_texts=True,
170
+ use_text_denoiser=False,
171
+ )
172
+ pages = doc.load() # pages is a list of `DocumentPage` objects
173
+
174
+ # 2️⃣ Initialise a tokenizer
175
+ tokenizer = AutoTokenizer.from_pretrained("radlab/polish-fast-tokenizer")
176
+
177
+ # 3️⃣ Load a dataset (JSON or JSONL) and automatically cache tokenisation
178
+ loader = DatasetLoader(
179
+ dataset_path="data/my_dataset",
180
+ tokenizer=tokenizer,
181
+ split_train_valid_test="0.8:0.1", # 80 % train, 10 % validation, 10 % test
182
+ pre_shuffle=True,
183
+ )
184
+ dataset = loader.load()
185
+
186
+ # 4️⃣ Access train / validation / test splits
187
+ train = dataset.train_dataset
188
+ valid = dataset.valid_dataset
189
+ test = dataset.test_dataset
190
+ ```
191
+
192
+ The resulting `train`, `valid`, and `test` objects are HuggingFace `datasets.Dataset` instances, ready to be passed to a
193
+ `transformers` trainer.
194
+
195
+ ---
196
+
197
+ ## Core Concepts
198
+
199
+ ### Dataset Types
200
+
201
+ | Class | Typical Use‑Case | Key Characteristics |
202
+ |-------------------------------|-------------------------------------------------|-------------------------------------------------------------------|
203
+ | `TextClassificationDataset` | Single‑label classification | Stores `text` and a scalar `label`. |
204
+ | `SequenceLabellingDataset` | NER / token‑level tagging | Generates IOB tags aligned to tokenised inputs. |
205
+ | `SequenceToSequenceDataset` | Translation, summarisation, etc. | Handles source‑target pairs (`text` ↔ `label`). |
206
+ | `SemanticSimilarityDataset` | Pairwise similarity / ranking | Provides `sentence1`, `sentence2`, and a similarity `score`. |
207
+ | `DPODataset` | Preference‑based learning (chosen vs. rejected) | Keeps raw prompts and two possible completions. |
208
+ | `DialogueAlternativesDataset` | Multi‑choice dialogue generation | Stores a shared context and several possible endings with scores. |
209
+
210
+ All dataset classes inherit from `DatasetBaseClass`, which implements caching, optional shuffling, and split handling.
211
+
212
+ ### Text Loaders
213
+
214
+ | Loader | Supported Extension | Behaviour |
215
+ |-------------------|---------------------|--------------------------------------------------------|
216
+ | `PDFLoader` | `.pdf` | One page per PDF page via `pypdf`. |
217
+ | `TXTLoader` | `.txt` | Whole file as one page, with encoding autodetection. |
218
+ | `MarkdownLoader` | `.md` | Reads like `TXTLoader`, optional markup stripping. |
219
+ | `DOCXLoader` | `.docx` | Parses paragraphs and tables via `python-docx`. |
220
+ | `HTMLLoader` | `.html` | Strips markup via BeautifulSoup, `split_by_paragraph`. |
221
+ | `InputTextLoader` | `.input_text` | Wraps a raw string supplied via options. |
222
+
223
+ Loaders share a common base (`LoaderI`) that handles:
224
+
225
+ - Tokeniser initialisation,
226
+ - Optional language detection,
227
+ - Text denoising,
228
+ - Chunking into token‑size windows.
229
+
230
+ Every loader returns `DocumentPage` objects (`page_content` plus a `metadata`
231
+ dictionary), so no document-loading framework is needed to use `radlab-data`.
232
+
233
+ Markdown files are read as plain text and every page is tagged with
234
+ `document_format: markdown`. Pass the `strip_markdown` option to remove markdown markup
235
+ (headings, emphasis, links, code fences, list markers) while keeping the underlying text:
236
+
237
+ ```python
238
+ from radlab_data.text.loaders.markdown import MarkdownLoader
239
+
240
+ loader = MarkdownLoader("docs/README.md", options={"strip_markdown": True})
241
+ pages = loader.load()
242
+ ```
243
+
244
+ ### Processing Pipeline
245
+
246
+ `radlab-data` ships with a lightweight pipeline framework:
247
+
248
+ ```python
249
+ from radlab_data.preprocessing.pipeline import Pipeline
250
+ from radlab_data.preprocessing.pipeline_modules import SplitLabels, RemoveDuplicates
251
+
252
+ pipeline = Pipeline(modules=[
253
+ SplitLabels(),
254
+ RemoveDuplicates(),
255
+ # add your custom PipelineElementBase subclasses here
256
+ ])
257
+
258
+ processed_dataset = pipeline(original_dataset)
259
+ ```
260
+
261
+ Each pipeline element inherits from `PipelineElementBase` and implements a `_process_data` method that receives a
262
+ dataset instance and returns the transformed instance. The framework automatically validates that the element supports
263
+ the dataset type.
264
+
265
+ ---
266
+
267
+ ## Command‑Line Utilities
268
+
269
+ The package currently does not ship command‑line entry points; dataset
270
+ conversion and preprocessing are driven from Python through the classes
271
+ described above.
272
+
273
+ ---
274
+
275
+ ## Configuration
276
+
277
+ All default settings live under `radlab_data/text/loaders/config.py`:
278
+
279
+ ```python
280
+ TOKENIZER_PATH = "radlab/polish-fast-tokenizer"
281
+ AVAILABLE_FILE_EXTENSIONS = ["txt", "md", "pdf", "docx", "html", "input_text"]
282
+ ```
283
+
284
+ You can override these values at runtime by passing the appropriate arguments to `Document` or the loader classes.
285
+
286
+ ---
287
+
288
+ ## License
289
+
290
+ `radlab-data` is released under the **Apache 2.0 License**. See the `LICENSE` file for full details.