hllpp 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
hllpp/Autocorrector.py ADDED
@@ -0,0 +1,689 @@
1
+ ############################################
2
+ # Copyright (c) 2026 Shun/修海 (@shun4midx) #
3
+ # Project: HyperLogLogPlusPlus-Autocorrect #
4
+ # File Type: Python file #
5
+ # File: Autocorrector.py #
6
+ ############################################
7
+
8
+ import math
9
+ import os
10
+ import time
11
+ import warnings
12
+ from dataclasses import dataclass
13
+ from typing import Dict, List, Sequence, Union
14
+
15
+ from .HyperLogLogPlusPlus import HyperLogLogPlusPlus, SketchConfig
16
+
17
+ SuggestionValue = Union[str, List[str]]
18
+ ScoreValue = Union[float, List[float]]
19
+
20
+ def extract_qgrams(word, q=2, fuzzier=None): # `fuzzier` is retained only for backwards compatibility
21
+ # Depreciation notice
22
+ if fuzzier is not None:
23
+ warnings.warn("`fuzzier` is retained only for backwards compatibility and has no effect in the HLL++ structural algorithm.", DeprecationWarning, stacklevel=2)
24
+
25
+ # Code
26
+ if len(word) < q:
27
+ return []
28
+
29
+ padded = f" {word} "
30
+
31
+ if q != 2:
32
+ return {padded[i : i + q] for i in range(len(padded) - q + 1)}
33
+
34
+ qgrams = []
35
+
36
+ for i in range(len(padded) - 1):
37
+ qgram = padded[i : i + 2]
38
+
39
+ qgrams.append(qgram)
40
+ qgrams.append(f"{qgram[1]}{qgram[0]}")
41
+
42
+ return qgrams
43
+
44
+
45
+ def extract_reversed_qgrams(word, q=2): # extract locally reversed q-gram features and also space variants
46
+ if len(word) < q:
47
+ return []
48
+
49
+ if q != 2:
50
+ return [word[i : i + q][::-1] for i in range(len(word) - q + 1)]
51
+
52
+ qgrams = []
53
+
54
+ for i in range(len(word) - 1):
55
+ left = word[i]
56
+ right = word[i + 1]
57
+
58
+ qgrams.append(f"{right}{left}")
59
+ qgrams.append(f"{right} ")
60
+ qgrams.append(f" {left}")
61
+
62
+ return qgrams
63
+
64
+ # ======== REPACKAGING ======== #
65
+
66
+ def is_valid(word, letters=None):
67
+ if letters is None:
68
+ return True
69
+
70
+ return all(character in letters for character in word.lower() )
71
+
72
+ def _read_source(src):
73
+ """
74
+ Read words from:
75
+ - a list or tuple;
76
+ - a file containing one word per line;
77
+ - or a single literal word.
78
+ """
79
+ if isinstance(src, (list, tuple)):
80
+ return [str(item) for item in src]
81
+
82
+ if not isinstance(src, str):
83
+ raise ValueError(f"`src` ({src!r}) must be a list/tuple, a file path, or a string")
84
+
85
+ if os.path.isfile(src):
86
+ with open(src, "r", encoding="utf-8") as file:
87
+ return [line.strip() for line in file if line.strip()]
88
+
89
+ # A string that looks like a path should not silently become one query.
90
+ if (src.endswith(".txt") or os.path.sep in src or (os.path.altsep is not None and os.path.altsep in src)):
91
+ raise FileNotFoundError(f"Input file not found: {os.path.abspath(src)}")
92
+
93
+ # Otherwise, treat it as one literal word.
94
+ return [src]
95
+
96
+ def load_words(src, letters=None):
97
+ raw = _read_source(src)
98
+ valid_raw = [word for word in raw if is_valid(word, letters)]
99
+
100
+ words = [word.lower() for word in valid_raw]
101
+ display = {word.lower(): word for word in valid_raw}
102
+
103
+ return words, display
104
+
105
+ def load_queries(src):
106
+ raw = _read_source(src)
107
+ return [(word, word.lower()) for word in raw]
108
+
109
+ addon_files = ["texting"] # Files to addon 20k_database.txt
110
+
111
+ @dataclass
112
+ class Results:
113
+ suggestions: Dict[str, SuggestionValue]
114
+ scores: Dict[str, ScoreValue]
115
+
116
+ class Autocorrector:
117
+ # The algorithm basically relies on padded q-grams, a separate loose reversed channel, adjacent-transposition rescue, and keyboard-aware edit distance.
118
+ # Default non-custom imported keyboards here would disregarded special characters (such as commas, not things like é and ö). Please import your own if you need to. I consider number rows too.
119
+ # Of course, Dvorak is not as intuitive. I replaced special characters with a whitespace for sake of consistency.
120
+ # "a-z" only considers English letters. For French, for example, you can import valid_letters = ["a-z", "é", "É", "à", "À", "ê". "Ê". "è", "È"]
121
+ def __init__(self, dictionary_list=os.path.join("test_files", "20k_database.txt"), valid_letters="a-z", keyboard="qwerty", *, alpha=None, beta=0.85, b=10, shortlist_size=100, keyboard_shortlist_size=75, transposition_bonus=0.35, reversal_weight=0.80):
122
+ self.letters = self._build_valid_letter_set(valid_letters)
123
+ self.keyboard = self._build_keyboard(keyboard)
124
+ self.KEY_POS = self._build_key_positions(self.keyboard)
125
+ self.KEY_COST = self._build_key_costs(self.KEY_POS)
126
+
127
+ self.word_dict, self.display_map = (self._load_dictionary(dictionary_list))
128
+
129
+ self.alpha = alpha
130
+ if alpha is not None:
131
+ warnings.warn("`alpha` is retained only for backwards compatibility and has no effect in the HLL++ structural algorithm.", DeprecationWarning, stacklevel=2)
132
+
133
+ self.beta = float(beta)
134
+ self.b = int(b)
135
+ self.shortlist_size = int(shortlist_size)
136
+ self.keyboard_shortlist_size = int(keyboard_shortlist_size)
137
+ self.transposition_bonus = float(transposition_bonus)
138
+ self.reversal_weight = float(reversal_weight)
139
+
140
+ if self.shortlist_size <= 0:
141
+ raise ValueError("`shortlist_size` must be positive")
142
+
143
+ if self.keyboard_shortlist_size <= 0:
144
+ raise ValueError("`keyboard_shortlist_size` must be positive")
145
+
146
+ if not 0.0 <= self.reversal_weight <= 1.0:
147
+ raise ValueError("`reversal_weight` must be between 0 and 1")
148
+
149
+ self.removed_words = set()
150
+ self.compact_threshold = 0.1
151
+
152
+ self.save_dictionary()
153
+
154
+ @staticmethod
155
+ def _build_valid_letter_set(valid_letters):
156
+ if valid_letters in (None, "", []):
157
+ return None
158
+
159
+ if isinstance(valid_letters, str):
160
+ valid_letters = [valid_letters]
161
+ elif not isinstance(valid_letters, list):
162
+ raise ValueError(f"`valid_letters` ({valid_letters!r}) must be a string or a list")
163
+
164
+ letters = []
165
+
166
+ for letter in valid_letters:
167
+ if letter == "a-z":
168
+ letters.extend(chr(ord("a") + i) for i in range(26))
169
+ elif letter == "0-9":
170
+ letters.extend(chr(ord("0") + i) for i in range(10))
171
+ elif (isinstance(letter, str) and len(letter) == 1 and letter != " "):
172
+ letters.append(letter.lower())
173
+ else:
174
+ raise ValueError('''`valid_letters` must contain single non-space characters or the abbreviations "a-z" and "0-9"''')
175
+
176
+ return set(letters)
177
+
178
+ @staticmethod
179
+ def _build_keyboard(keyboard):
180
+ presets = {
181
+ "qwerty": ["1234567890", "qwertyuiop", "asdfghjkl", "zxcvbnm"],
182
+ "azerty": ["1234567890", "azertyuiop", "qsdfghjklm", "wxcvbn"],
183
+ "qwertz": ["1234567890", "qwertzuiopü", "asdfghjklöä", "yxcvbnm"],
184
+ "dvorak": ["1234567890", "' pyfgcrl", "aoeuidhtns", " qjkxbmwvz"],
185
+ "colemak": ["1234567890", "qwfpgjluy", "arstdhneio", "zxcvbkm"],
186
+ }
187
+
188
+ if isinstance(keyboard, str):
189
+ if keyboard not in presets:
190
+ raise ValueError('''`keyboard` must be one of "qwerty", "azerty", "qwertz", "dvorak", or "colemak", or a custom list of rows''')
191
+ return presets[keyboard]
192
+
193
+ if not isinstance(keyboard, list) or not all(isinstance(row, str) for row in keyboard):
194
+ raise ValueError("`keyboard` must be a preset name or a list of row strings")
195
+
196
+ return keyboard
197
+
198
+ @staticmethod
199
+ def _build_key_positions(keyboard):
200
+ positions = {}
201
+
202
+ for row_index, row in enumerate(keyboard):
203
+ for column_index, character in enumerate(row):
204
+ positions[character] = (row_index, column_index)
205
+
206
+ return positions
207
+
208
+ @staticmethod
209
+ def _build_key_costs(key_positions):
210
+ costs = {}
211
+
212
+ for a, (xa, ya) in key_positions.items():
213
+ for b, (xb, yb) in key_positions.items():
214
+ costs[(a, b)] = math.sqrt((xa - xb)**2 + (ya - yb)**2)
215
+ return costs
216
+
217
+ def _load_dictionary(self, dictionary_list):
218
+ # 1) Already a Python sequence?
219
+ if isinstance(dictionary_list, (list, tuple)):
220
+ return load_words(dictionary_list, self.letters)
221
+
222
+ # 2) File on disk?
223
+ if dictionary_list in addon_files:
224
+ base_dir = os.path.dirname(os.path.abspath(__file__))
225
+ base_path = os.path.join(base_dir, "test_files", "20k_database.txt")
226
+ addon_path = os.path.join(base_dir, "test_files", f"{dictionary_list}.txt")
227
+ combined_words = (_read_source(base_path) + _read_source(addon_path))
228
+ return load_words(combined_words, self.letters)
229
+
230
+ if isinstance(dictionary_list, str) and os.path.isfile(dictionary_list):
231
+ return load_words(os.path.abspath(dictionary_list), self.letters)
232
+
233
+ # 3) String?
234
+ if not isinstance(dictionary_list, str):
235
+ raise ValueError("`dictionary_list` must be a list/tuple, a path, or a known addon")
236
+
237
+ base_dir = os.path.dirname(os.path.abspath(__file__))
238
+ dictionary_path = os.path.join(base_dir, dictionary_list)
239
+
240
+ if not os.path.isfile(dictionary_path):
241
+ raise FileNotFoundError(f"Dictionary file not found: {dictionary_path}")
242
+
243
+ return load_words(dictionary_path, self.letters)
244
+
245
+ def key_dist(self, a, b):
246
+ if a == b:
247
+ return 0.0
248
+ return self.KEY_COST.get((a, b), 1.0)
249
+
250
+ def word_dist(self, a, b): # Keyboard-aware Levenshtein
251
+ na = len(a)
252
+ nb = len(b)
253
+
254
+ previous = [float(j) for j in range(nb + 1)]
255
+ current = [0.0] * (nb + 1)
256
+
257
+ for i in range(1, na + 1):
258
+ current[0] = float(i)
259
+ ai = a[i - 1]
260
+
261
+ for j in range(1, nb + 1):
262
+ bj = b[j - 1]
263
+
264
+ substitution = previous[j - 1] + self.key_dist(ai, bj)
265
+ deletion = previous[j] + 1.0
266
+ insertion = current[j - 1] + 1.0
267
+ current[j] = min(substitution, deletion, insertion)
268
+
269
+ previous, current = current, previous
270
+
271
+ return previous[nb]
272
+
273
+ def is_valid(self, word):
274
+ return is_valid(word, self.letters)
275
+
276
+ def build_query_sketch(self, qgrams):
277
+ sketch = HyperLogLogPlusPlus(self.cfg)
278
+
279
+ for gram in set(qgrams):
280
+ sketch.insert(f"feature:{gram}")
281
+
282
+ return sketch
283
+
284
+ @staticmethod
285
+ def _score_from_sizes(intersection, left_size):
286
+ if left_size <= 0.0:
287
+ return 0.0
288
+ score = intersection / left_size
289
+
290
+ return min(max(score, 0.0), 1.0)
291
+
292
+ def save_dictionary(self):
293
+ self.t0 = time.perf_counter()
294
+ self.q = 2
295
+ self.cfg = SketchConfig(b=self.b, sparse=True)
296
+ self.word_sketches = []
297
+ self.word_estimates = []
298
+
299
+ self.qgram_word_indices = {}
300
+
301
+ # WORD_COUNT tracks physically stored entries, ACTIVE_WORD_COUNT excludes lazily removed entries. Between compactions, indices remain stable because word_dict and the parallel sketch arrays are append-only.
302
+ self.WORD_COUNT = len(self.word_dict)
303
+ self.ACTIVE_WORD_COUNT = self.WORD_COUNT - len(self.removed_words)
304
+ self.word_to_idx = {word: idx for idx, word in enumerate(self.word_dict)}
305
+
306
+ if self.WORD_COUNT == 0:
307
+ raise ValueError("Dictionary cannot be empty")
308
+
309
+ for idx, word in enumerate(self.word_dict):
310
+ qgrams = set(extract_qgrams(word, self.q))
311
+ word_sketch = HyperLogLogPlusPlus(self.cfg)
312
+
313
+ for gram in qgrams:
314
+ self.qgram_word_indices.setdefault(gram, []).append(idx)
315
+ word_sketch.insert(f"feature:{gram}")
316
+
317
+ self.word_sketches.append(word_sketch)
318
+ self.word_estimates.append(word_sketch.estimate())
319
+
320
+ self.preprocessing_time = time.perf_counter() - self.t0
321
+
322
+ def _append_word(self, word, display):
323
+ # Append one new word w/o rebuilding the dict. This is proportional to number of q-gram freatures generated from the word, i.e. O(L) for fixed q, where L = word len
324
+ idx = len(self.word_dict)
325
+
326
+ self.word_dict.append(word)
327
+ self.display_map[word] = display
328
+ self.word_to_idx[word] = idx
329
+
330
+ qgrams = set(extract_qgrams(word, self.q))
331
+ word_sketch = HyperLogLogPlusPlus(self.cfg)
332
+
333
+ for gram in qgrams:
334
+ self.qgram_word_indices.setdefault(gram, []).append(idx)
335
+ word_sketch.insert(f"feature:{gram}")
336
+
337
+ self.word_sketches.append(word_sketch)
338
+ self.word_estimates.append(word_sketch.estimate())
339
+
340
+ self.WORD_COUNT += 1
341
+ self.ACTIVE_WORD_COUNT += 1
342
+
343
+ def add_dictionary(self, to_be_added): # Overall cost is O(L), L = input len
344
+ words, displays = load_words(to_be_added, self.letters)
345
+ added = []
346
+ restored = []
347
+
348
+ for word in words:
349
+ idx = self.word_to_idx.get(word)
350
+ if idx is not None:
351
+ if word in self.removed_words:
352
+ self.removed_words.remove(word)
353
+ self.ACTIVE_WORD_COUNT += 1
354
+ self.display_map[word] = displays.get(word, self.display_map.get(word, word))
355
+ restored.append(word)
356
+ continue
357
+
358
+ self._append_word(word, displays.get(word, word))
359
+ added.append(word)
360
+
361
+ return added + restored
362
+
363
+ def remove_dictionary(self, to_be_removed): # Lazily remove using tombstones, normal removal is O(1), O(L) amortized
364
+ words, _ = load_words(to_be_removed, self.letters)
365
+ removed = []
366
+
367
+ for word in words:
368
+ if word in self.word_to_idx and word not in self.removed_words:
369
+ self.removed_words.add(word)
370
+ self.ACTIVE_WORD_COUNT -= 1
371
+ removed.append(word)
372
+
373
+ # Rebuild only after a const frac of tombstones, so O(NL) rebuild is spread across Theta(N) deletions
374
+ if self.ACTIVE_WORD_COUNT > 0 and self.word_dict and len(self.removed_words) >= len(self.word_dict) * self.compact_threshold:
375
+ self.word_dict = [word for word in self.word_dict if word not in self.removed_words]
376
+ self.display_map = {word: self.display_map[word] for word in self.word_dict}
377
+ self.removed_words.clear()
378
+ self.save_dictionary()
379
+
380
+ return removed
381
+
382
+ def _candidate_indices(self, query_qgrams):
383
+ overlap_counts = {}
384
+ removed_words = self.removed_words
385
+ word_dict = self.word_dict
386
+
387
+ for gram in query_qgrams:
388
+ for idx in self.qgram_word_indices.get(gram, ()):
389
+ if word_dict[idx] in removed_words:
390
+ continue
391
+
392
+ overlap_counts[idx] = overlap_counts.get(idx, 0) + 1
393
+
394
+ candidates = list(overlap_counts.items())
395
+ candidates.sort(key=lambda item: (-item[1], item[0]))
396
+ return candidates[:self.shortlist_size]
397
+
398
+ @staticmethod
399
+ def _is_adjacent_transposition(query, candidate):
400
+ if len(query) != len(candidate):
401
+ return False
402
+
403
+ mismatches = [i for i, (a, b) in enumerate(zip(query, candidate)) if a != b]
404
+ if len(mismatches) != 2:
405
+ return False
406
+ i, j = mismatches
407
+
408
+ return j == i + 1 and query[i] == candidate[j] and query[j] == candidate[i]
409
+
410
+ def _rank_candidates(self, query, use_keyboard, print_details=False):
411
+ retrieval_qgrams = set(extract_qgrams(query, self.q))
412
+ scoring_qgrams = retrieval_qgrams
413
+ reversed_scoring_qgrams = set(extract_reversed_qgrams(query, self.q))
414
+
415
+ if print_details:
416
+ print(f"Query {query!r}: features={len(scoring_qgrams)}, reversed_features={len(reversed_scoring_qgrams)}")
417
+ print("Structural q-grams: " + ", ".join(repr(gram) for gram in sorted(scoring_qgrams)))
418
+ print("Loose reversed q-grams: " + ", ".join(repr(gram) for gram in sorted(reversed_scoring_qgrams)))
419
+
420
+ query_sketch = self.build_query_sketch(scoring_qgrams)
421
+ query_estimate = query_sketch.estimate()
422
+
423
+ reversed_query_sketch = self.build_query_sketch(reversed_scoring_qgrams)
424
+ reversed_query_estimate = (reversed_query_sketch.estimate())
425
+
426
+ candidate_indices = self._candidate_indices(retrieval_qgrams)
427
+ retrieved_count = len(candidate_indices)
428
+ candidate_map = dict(candidate_indices)
429
+
430
+ for i in range(len(query) - 1):
431
+ if query[i] == query[i + 1]:
432
+ continue
433
+
434
+ swapped = query[:i] + query[i + 1] + query[i] + query[i + 2:]
435
+
436
+ idx = self.word_to_idx.get(swapped)
437
+ if idx is not None and swapped not in self.removed_words:
438
+ candidate_map.setdefault(idx, 0)
439
+
440
+ candidate_indices = list(candidate_map.items())
441
+
442
+ if print_details:
443
+ rescued_count =len(candidate_indices) - retrieved_count
444
+ print(f"Candidates: {retrieved_count} retrieved, {rescued_count} transposition-rescued, {len(candidate_indices)} total")
445
+
446
+ preliminary = []
447
+
448
+ for idx, exact_overlap in candidate_indices:
449
+ word_sketch = self.word_sketches[idx]
450
+ word_estimate = self.word_estimates[idx]
451
+
452
+ structural_union = (query_sketch.union_estimate(word_sketch))
453
+ structural_intersection = max(0.0, query_estimate + word_estimate - structural_union)
454
+ structural_intersection = min(structural_intersection, query_estimate, word_estimate)
455
+
456
+ normal_structural_score = self._score_from_sizes(intersection=structural_intersection, left_size=query_estimate)
457
+ reversed_union = reversed_query_sketch.union_estimate(word_sketch)
458
+ reversed_intersection = max(0.0, reversed_query_estimate + word_estimate - reversed_union)
459
+ reversed_intersection = min(reversed_intersection, reversed_query_estimate, word_estimate)
460
+ reversed_structural_score = self._score_from_sizes(intersection=reversed_intersection,left_size=reversed_query_estimate)
461
+
462
+ structural_score = min(1.0, normal_structural_score + self.reversal_weight * max(0.0, reversed_structural_score - normal_structural_score))
463
+
464
+ candidate = self.word_dict[idx]
465
+
466
+ length_ratio = abs(len(candidate) - len(query)) / max(len(query), 1)
467
+ length_score = max(0.0, 1.0 - length_ratio**2)
468
+ exact_bonus = 1.0 if query == candidate else 0.0
469
+
470
+ preliminary_score = structural_score * length_score + exact_bonus
471
+ preliminary.append((preliminary_score, idx, structural_score, exact_overlap, length_score, normal_structural_score, reversed_structural_score, word_estimate, structural_union, structural_intersection, reversed_union, reversed_intersection))
472
+
473
+ preliminary.sort(key=lambda item: (-item[0], item[1]))
474
+
475
+ keyboard_limit = min(self.keyboard_shortlist_size, len(preliminary))
476
+
477
+ ranked = []
478
+
479
+ for position, (preliminary_score, idx, structural_score, exact_overlap, length_score, normal_structural_score, reversed_structural_score, word_estimate, structural_union, structural_intersection, reversed_union, reversed_intersection) in enumerate(preliminary):
480
+ candidate = self.word_dict[idx]
481
+
482
+ if use_keyboard and position < keyboard_limit:
483
+ keyboard_score = 1.0 / (1.0 + self.word_dist(query, self.word_dict[idx]))
484
+ else:
485
+ keyboard_score = 0.0
486
+
487
+ exact_bonus = 1.0 if query == self.word_dict[idx] else 0.0
488
+ transposition_bonus = self.transposition_bonus if self._is_adjacent_transposition(query, candidate) else 0.0
489
+
490
+ score = structural_score * length_score + self.beta * keyboard_score + transposition_bonus + exact_bonus
491
+
492
+ ranked.append((score, idx, structural_score, exact_overlap, normal_structural_score, reversed_structural_score, length_score, keyboard_score, transposition_bonus, exact_bonus, word_estimate, structural_union, structural_intersection, reversed_union, reversed_intersection))
493
+
494
+ ranked.sort(key=lambda item: (-item[0], item[1]))
495
+
496
+ if print_details:
497
+ print("Top ranked candidates:")
498
+
499
+ for (score, idx, structural_score, exact_overlap, normal_structural_score, reversed_structural_score, length_score, keyboard_score, transposition_bonus, exact_bonus, word_estimate, structural_union, structural_intersection, reversed_union, reversed_intersection) in ranked[:5]:
500
+ candidate = self.word_dict[idx]
501
+ print(f" {candidate!r}:")
502
+ print(f" retrieval overlap: {exact_overlap}")
503
+ print(f" HLL++ normal: query={query_estimate:.6f}, candidate={word_estimate:.6f}, union={structural_union:.6f}, intersection={structural_intersection:.6f}")
504
+ print(f" HLL++ reversed: query={reversed_query_estimate:.6f}, candidate={word_estimate:.6f}, union={reversed_union:.6f}, intersection={reversed_intersection:.6f}")
505
+ print(f" structural: normal={normal_structural_score:.6f}, reversed={reversed_structural_score:.6f}, blended={structural_score:.6f}")
506
+
507
+ structural_contribution = structural_score * length_score
508
+ keyboard_contribution = self.beta * keyboard_score
509
+ print(f" modifiers: length={length_score:.6f}, keyboard={keyboard_score:.6f}, transpose_bonus={transposition_bonus:.3f}, exact_bonus={exact_bonus:.1f}")
510
+ print(f" contributions: structural*length={structural_contribution:.6f}, beta*keyboard={keyboard_contribution:.6f}")
511
+ print(f" final score: {score:.6f}")
512
+
513
+ return ranked
514
+
515
+ def autocorrect(self, queries_list, output_file="None", use_keyboard=True, return_invalid_words=True, print_details=False, print_times=False):
516
+ if print_times:
517
+ self.save_dictionary()
518
+
519
+ queries = load_queries(queries_list)
520
+
521
+ self.t2 = time.perf_counter()
522
+
523
+ output = []
524
+ suggestions = {}
525
+ final_scores = {}
526
+
527
+ for query_display, query in queries:
528
+ if not self.is_valid(query):
529
+ replacement = query_display if return_invalid_words else ""
530
+ suggestions[query_display] = replacement
531
+ final_scores[query_display] = 0.0
532
+ output.append(replacement)
533
+ continue
534
+
535
+ ranked = self._rank_candidates(query, use_keyboard=use_keyboard, print_details=print_details)
536
+
537
+ if not ranked:
538
+ replacement = query_display if return_invalid_words else ""
539
+ suggestions[query_display] = replacement
540
+ final_scores[query_display] = 0.0
541
+ output.append(replacement)
542
+ continue
543
+
544
+ (best_score, best_idx, best_structural_score, best_exact_overlap, *_) = ranked[0]
545
+
546
+ picked = self.word_dict[best_idx]
547
+ displayed_picked = self.display_map.get(picked, picked)
548
+
549
+ if print_details:
550
+ print(f"Selected {displayed_picked!r} for {query_display!r}: final={best_score:.6f}, containment={best_structural_score:.6f}, retrieval_overlap={best_exact_overlap}")
551
+ print("-" * 60)
552
+
553
+ suggestions[query_display] = displayed_picked
554
+ final_scores[query_display] = best_score
555
+ output.append(displayed_picked)
556
+
557
+ self.t3 = time.perf_counter()
558
+
559
+ if output_file != "None":
560
+ with open(output_file, "w", encoding="utf-8") as output_stream:
561
+ output_stream.write("\n".join(output))
562
+
563
+ if print_times:
564
+ print(f"Dictionary preprocessing: {self.preprocessing_time:.3f}s")
565
+ print(f"Current query batch: {self.t3 - self.t2:.3f}s")
566
+ print(f"Total autocorrect: {self.preprocessing_time + self.t3 - self.t2:.3f}s")
567
+
568
+ return Results(suggestions=suggestions, scores=final_scores)
569
+
570
+ def top_k(self, queries_list, k, output_file="None", use_keyboard=True, return_invalid_words=True, print_details=False, print_times=False):
571
+ if not isinstance(k, int) or isinstance(k, bool):
572
+ raise TypeError("`k` must be an integer")
573
+
574
+ if k <= 0:
575
+ raise ValueError("`k` must be positive")
576
+
577
+ if print_times:
578
+ self.save_dictionary()
579
+
580
+ queries = load_queries(queries_list)
581
+
582
+ self.t2 = time.perf_counter()
583
+
584
+ output = []
585
+ suggestions = {}
586
+ final_scores = {}
587
+
588
+ for query_display, query in queries:
589
+ if not self.is_valid(query):
590
+ if return_invalid_words:
591
+ top_words = [query_display]
592
+ else:
593
+ top_words = []
594
+
595
+ while len(top_words) < k:
596
+ top_words.append("")
597
+
598
+ top_scores = [0.0] * k
599
+
600
+ suggestions[query_display] = top_words
601
+ final_scores[query_display] = top_scores
602
+ output.append(" ".join(top_words))
603
+ continue
604
+
605
+ ranked = self._rank_candidates(query, use_keyboard=use_keyboard, print_details=print_details)
606
+
607
+ seen = set()
608
+ top_words = []
609
+ top_scores = []
610
+
611
+ for (score, idx, *_) in ranked:
612
+ suggestion = self.display_map.get(self.word_dict[idx], self.word_dict[idx])
613
+
614
+ if suggestion in seen:
615
+ continue
616
+
617
+ seen.add(suggestion)
618
+ top_words.append(suggestion)
619
+ top_scores.append(score)
620
+
621
+ if len(top_words) == k:
622
+ break
623
+
624
+ if not top_words and return_invalid_words:
625
+ top_words.append(query_display)
626
+ top_scores.append(0.0)
627
+
628
+ while len(top_words) < k:
629
+ top_words.append("")
630
+ top_scores.append(0.0)
631
+
632
+ if print_details:
633
+ displayed_results = [f"{word!r} ({score:.6f})" for word, score in zip(top_words, top_scores) if word]
634
+ print(f"Selected top {k} for {query_display!r}: {', '.join(displayed_results)}")
635
+ print("-" * 60)
636
+
637
+ suggestions[query_display] = top_words
638
+ final_scores[query_display] = top_scores
639
+ output.append(" ".join(top_words))
640
+
641
+ self.t3 = time.perf_counter()
642
+
643
+ if output_file != "None":
644
+ with open(output_file, "w", encoding="utf-8") as output_stream:
645
+ output_stream.write("\n".join(output))
646
+
647
+ if print_times:
648
+ print(f"Dictionary preprocessing: {self.preprocessing_time:.3f}s")
649
+ print(f"Current query batch: {self.t3 - self.t2:.3f}s")
650
+ print(f"Total top-{k}: {self.preprocessing_time + self.t3 - self.t2:.3f}s")
651
+
652
+ return Results(suggestions=suggestions, scores=final_scores)
653
+
654
+ def top3(self, queries_list, output_file="None", use_keyboard=True, return_invalid_words=True, print_details=False, print_times=False):
655
+ return self.top_k(queries_list=queries_list, k=3, output_file=output_file, use_keyboard=use_keyboard, return_invalid_words=return_invalid_words, print_details=print_details, print_times=print_times)
656
+
657
+ # ======== SAMPLE USAGE ======== #
658
+ if __name__ == "__main__":
659
+ ac = Autocorrector()
660
+
661
+ # File
662
+ ans1 = ac.autocorrect("test_files/typo_file.txt", "outputs/class_suggestions.txt")
663
+ print(ans1.suggestions)
664
+ print(ans1.scores)
665
+
666
+ ans2 = ac.top3("test_files/typo_file.txt", "outputs/class_suggestions.txt")
667
+
668
+ # Or even top 5
669
+ ans2_top5 = ac.top_k("test_files/typo_file.txt", 5, "outputs/class_suggestions.txt")
670
+
671
+ # Optionally, you can not want it to output it into a file, then:
672
+ # Individual strings
673
+ ans3 = ac.autocorrect("hillo")
674
+ ans4 = ac.top3("hillo")
675
+ ans4_top5 = ac.top_k("hillo", 5)
676
+
677
+ # Arrays
678
+ ans5 = ac.autocorrect(["tsetign", "hillo", "goobye", "haedhpoesn"])
679
+ ans6 = ac.top3(["tsetign", "hillo", "goobye", "haedhpoesn"])
680
+
681
+ # You can even have a custom dictionary!
682
+ dictionary = ["apple", "banana", "grape", "orange"]
683
+ custom_ac = Autocorrector(dictionary)
684
+
685
+ ans7 = custom_ac.autocorrect(["applle", "banana", "banan", "orenge", "grap", "pineapple"])
686
+ ans8 = custom_ac.top3(["applle", "banana", "banan", "orenge", "grap", "pineapple"])
687
+
688
+ print(ans7.suggestions)
689
+ print(ans8.suggestions)