urdu-text-utils 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/README.md +18 -2
- package/dist/index.cjs +305 -4
- package/dist/index.cjs.map +1 -1
- package/dist/index.d.cts +64 -4
- package/dist/index.d.ts +64 -4
- package/dist/index.js +302 -5
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/dist/index.d.cts
CHANGED
|
@@ -110,9 +110,29 @@ declare function hasUrduSpecificLetters(input: string): boolean;
|
|
|
110
110
|
declare function countWords(input: string): number;
|
|
111
111
|
/** The word list behind {@link countWords}. Useful for tokenizing before search or indexing. */
|
|
112
112
|
declare function splitWords(input: string): string[];
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
interface SplitSentenceOptions {
|
|
114
|
+
/** If true, the sentence-ending punctuation (۔ ؟ ! . etc.) is preserved with each sentence. Default `false`. */
|
|
115
|
+
preserveTerminators?: boolean;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Sentences count, split on Urdu and standard terminators (۔ ؟ ! . …).
|
|
119
|
+
* Protects common titles, abbreviations, and numeric decimals from false splits.
|
|
120
|
+
*/
|
|
121
|
+
declare function countSentences(input: string, options?: SplitSentenceOptions): number;
|
|
122
|
+
/**
|
|
123
|
+
* Split text into sentences using Urdu punctuation rules.
|
|
124
|
+
*
|
|
125
|
+
* Handles Urdu full stop `۔`, Arabic question mark `؟`, exclamation `!`,
|
|
126
|
+
* ASCII `.`, `?`, `!`, and ellipses `…`, while protecting abbreviations and numbers.
|
|
127
|
+
*
|
|
128
|
+
* @param input - Input text.
|
|
129
|
+
* @param options - Options controlling termination preservation.
|
|
130
|
+
*
|
|
131
|
+
* @example
|
|
132
|
+
* splitSentences("پاکستان ایک خوبصورت ملک ہے۔ اس کی تاریخ پرانی ہے۔")
|
|
133
|
+
* // ["پاکستان ایک خوبصورت ملک ہے", "اس کی تاریخ پرانی ہے"]
|
|
134
|
+
*/
|
|
135
|
+
declare function splitSentences(input: string, options?: SplitSentenceOptions): string[];
|
|
116
136
|
interface UrduStats {
|
|
117
137
|
/** Every codepoint, including spaces and diacritics. */
|
|
118
138
|
characters: number;
|
|
@@ -137,6 +157,46 @@ interface UrduStats {
|
|
|
137
157
|
*/
|
|
138
158
|
declare function analyzeUrdu(input: string): UrduStats;
|
|
139
159
|
|
|
160
|
+
/**
|
|
161
|
+
* Standard list of Urdu stop words.
|
|
162
|
+
*
|
|
163
|
+
* Covers pronouns, postpositions, auxiliaries, conjunctions, particles,
|
|
164
|
+
* and high-frequency functional words used across Urdu texts.
|
|
165
|
+
* All keys are canonical normalized Urdu.
|
|
166
|
+
*/
|
|
167
|
+
declare const URDU_STOP_WORDS: Set<string>;
|
|
168
|
+
/**
|
|
169
|
+
* Checks if a given Urdu word is a stop word.
|
|
170
|
+
*
|
|
171
|
+
* @param word - Word to test.
|
|
172
|
+
* @param customStopWords - Optional custom stop words set or array. Defaults to {@link URDU_STOP_WORDS}.
|
|
173
|
+
*
|
|
174
|
+
* @example
|
|
175
|
+
* isStopWord("اور") // true
|
|
176
|
+
* isStopWord("کتاب") // false
|
|
177
|
+
*/
|
|
178
|
+
declare function isStopWord(word: string, customStopWords?: Set<string> | string[]): boolean;
|
|
179
|
+
/**
|
|
180
|
+
* Filters out stop words from an array of words.
|
|
181
|
+
*
|
|
182
|
+
* @param words - Array of words to filter.
|
|
183
|
+
* @param customStopWords - Optional custom stop words set or array.
|
|
184
|
+
*
|
|
185
|
+
* @example
|
|
186
|
+
* filterStopWords(["یہ", "ایک", "اچھی", "کتاب", "ہے"]) // ["اچھی", "کتاب"]
|
|
187
|
+
*/
|
|
188
|
+
declare function filterStopWords(words: string[], customStopWords?: Set<string> | string[]): string[];
|
|
189
|
+
/**
|
|
190
|
+
* Removes stop words from an Urdu text string, returning the cleaned text.
|
|
191
|
+
*
|
|
192
|
+
* @param text - Input Urdu text.
|
|
193
|
+
* @param customStopWords - Optional custom stop words set or array.
|
|
194
|
+
*
|
|
195
|
+
* @example
|
|
196
|
+
* removeStopWords("یہ ایک بہترین کتاب ہے") // "بہترین کتاب"
|
|
197
|
+
*/
|
|
198
|
+
declare function removeStopWords(text: string, customStopWords?: Set<string> | string[]): string;
|
|
199
|
+
|
|
140
200
|
/**
|
|
141
201
|
* Comparator for Urdu strings, usable directly in `Array.prototype.sort`.
|
|
142
202
|
* Diacritics and Unicode variants are folded first, so spelling noise does not
|
|
@@ -242,4 +302,4 @@ interface SlugOptions {
|
|
|
242
302
|
*/
|
|
243
303
|
declare function urduSlug(input: string, options?: SlugOptions): string;
|
|
244
304
|
|
|
245
|
-
export { type DigitStyle, type IsUrduOptions, type NormalizeOptions, type SearchOptions, type SearchResult, type SlugOptions, type SortOptions, type UrduStats, analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
|
305
|
+
export { type DigitStyle, type IsUrduOptions, type NormalizeOptions, type SearchOptions, type SearchResult, type SlugOptions, type SortOptions, type SplitSentenceOptions, URDU_STOP_WORDS, type UrduStats, analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, filterStopWords, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isStopWord, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, removeStopWords, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
package/dist/index.d.ts
CHANGED
|
@@ -110,9 +110,29 @@ declare function hasUrduSpecificLetters(input: string): boolean;
|
|
|
110
110
|
declare function countWords(input: string): number;
|
|
111
111
|
/** The word list behind {@link countWords}. Useful for tokenizing before search or indexing. */
|
|
112
112
|
declare function splitWords(input: string): string[];
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
113
|
+
interface SplitSentenceOptions {
|
|
114
|
+
/** If true, the sentence-ending punctuation (۔ ؟ ! . etc.) is preserved with each sentence. Default `false`. */
|
|
115
|
+
preserveTerminators?: boolean;
|
|
116
|
+
}
|
|
117
|
+
/**
|
|
118
|
+
* Sentences count, split on Urdu and standard terminators (۔ ؟ ! . …).
|
|
119
|
+
* Protects common titles, abbreviations, and numeric decimals from false splits.
|
|
120
|
+
*/
|
|
121
|
+
declare function countSentences(input: string, options?: SplitSentenceOptions): number;
|
|
122
|
+
/**
|
|
123
|
+
* Split text into sentences using Urdu punctuation rules.
|
|
124
|
+
*
|
|
125
|
+
* Handles Urdu full stop `۔`, Arabic question mark `؟`, exclamation `!`,
|
|
126
|
+
* ASCII `.`, `?`, `!`, and ellipses `…`, while protecting abbreviations and numbers.
|
|
127
|
+
*
|
|
128
|
+
* @param input - Input text.
|
|
129
|
+
* @param options - Options controlling termination preservation.
|
|
130
|
+
*
|
|
131
|
+
* @example
|
|
132
|
+
* splitSentences("پاکستان ایک خوبصورت ملک ہے۔ اس کی تاریخ پرانی ہے۔")
|
|
133
|
+
* // ["پاکستان ایک خوبصورت ملک ہے", "اس کی تاریخ پرانی ہے"]
|
|
134
|
+
*/
|
|
135
|
+
declare function splitSentences(input: string, options?: SplitSentenceOptions): string[];
|
|
116
136
|
interface UrduStats {
|
|
117
137
|
/** Every codepoint, including spaces and diacritics. */
|
|
118
138
|
characters: number;
|
|
@@ -137,6 +157,46 @@ interface UrduStats {
|
|
|
137
157
|
*/
|
|
138
158
|
declare function analyzeUrdu(input: string): UrduStats;
|
|
139
159
|
|
|
160
|
+
/**
|
|
161
|
+
* Standard list of Urdu stop words.
|
|
162
|
+
*
|
|
163
|
+
* Covers pronouns, postpositions, auxiliaries, conjunctions, particles,
|
|
164
|
+
* and high-frequency functional words used across Urdu texts.
|
|
165
|
+
* All keys are canonical normalized Urdu.
|
|
166
|
+
*/
|
|
167
|
+
declare const URDU_STOP_WORDS: Set<string>;
|
|
168
|
+
/**
|
|
169
|
+
* Checks if a given Urdu word is a stop word.
|
|
170
|
+
*
|
|
171
|
+
* @param word - Word to test.
|
|
172
|
+
* @param customStopWords - Optional custom stop words set or array. Defaults to {@link URDU_STOP_WORDS}.
|
|
173
|
+
*
|
|
174
|
+
* @example
|
|
175
|
+
* isStopWord("اور") // true
|
|
176
|
+
* isStopWord("کتاب") // false
|
|
177
|
+
*/
|
|
178
|
+
declare function isStopWord(word: string, customStopWords?: Set<string> | string[]): boolean;
|
|
179
|
+
/**
|
|
180
|
+
* Filters out stop words from an array of words.
|
|
181
|
+
*
|
|
182
|
+
* @param words - Array of words to filter.
|
|
183
|
+
* @param customStopWords - Optional custom stop words set or array.
|
|
184
|
+
*
|
|
185
|
+
* @example
|
|
186
|
+
* filterStopWords(["یہ", "ایک", "اچھی", "کتاب", "ہے"]) // ["اچھی", "کتاب"]
|
|
187
|
+
*/
|
|
188
|
+
declare function filterStopWords(words: string[], customStopWords?: Set<string> | string[]): string[];
|
|
189
|
+
/**
|
|
190
|
+
* Removes stop words from an Urdu text string, returning the cleaned text.
|
|
191
|
+
*
|
|
192
|
+
* @param text - Input Urdu text.
|
|
193
|
+
* @param customStopWords - Optional custom stop words set or array.
|
|
194
|
+
*
|
|
195
|
+
* @example
|
|
196
|
+
* removeStopWords("یہ ایک بہترین کتاب ہے") // "بہترین کتاب"
|
|
197
|
+
*/
|
|
198
|
+
declare function removeStopWords(text: string, customStopWords?: Set<string> | string[]): string;
|
|
199
|
+
|
|
140
200
|
/**
|
|
141
201
|
* Comparator for Urdu strings, usable directly in `Array.prototype.sort`.
|
|
142
202
|
* Diacritics and Unicode variants are folded first, so spelling noise does not
|
|
@@ -242,4 +302,4 @@ interface SlugOptions {
|
|
|
242
302
|
*/
|
|
243
303
|
declare function urduSlug(input: string, options?: SlugOptions): string;
|
|
244
304
|
|
|
245
|
-
export { type DigitStyle, type IsUrduOptions, type NormalizeOptions, type SearchOptions, type SearchResult, type SlugOptions, type SortOptions, type UrduStats, analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
|
305
|
+
export { type DigitStyle, type IsUrduOptions, type NormalizeOptions, type SearchOptions, type SearchResult, type SlugOptions, type SortOptions, type SplitSentenceOptions, URDU_STOP_WORDS, type UrduStats, analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, filterStopWords, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isStopWord, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, removeStopWords, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
package/dist/index.js
CHANGED
|
@@ -298,12 +298,51 @@ function splitWords(input) {
|
|
|
298
298
|
if (!input) return [];
|
|
299
299
|
return input.split(WORD_SPLIT_RE).map((w) => w.trim()).filter((w) => w.length > 0);
|
|
300
300
|
}
|
|
301
|
-
|
|
302
|
-
|
|
301
|
+
var ABBREVIATIONS = [
|
|
302
|
+
"\u0688\u0627\u06A9\u0679\u0631",
|
|
303
|
+
"\u067E\u0631\u0648\u0641\u06CC\u0633\u0631",
|
|
304
|
+
"\u0627\u0646\u062C\u06CC\u0646\u0626\u0631",
|
|
305
|
+
"\u0627\u06CC\u0688\u0648\u0648\u06A9\u06CC\u0679",
|
|
306
|
+
"\u062C\u0646\u0627\u0628",
|
|
307
|
+
"\u0635\u0627\u062D\u0628",
|
|
308
|
+
"\u0635\u0627\u062D\u0628\u06C1",
|
|
309
|
+
"\u0645\u062D\u062A\u0631\u0645",
|
|
310
|
+
"\u0645\u062D\u062A\u0631\u0645\u06C1",
|
|
311
|
+
"\u0645\u0648\u0644\u0627\u0646\u0627",
|
|
312
|
+
"\u0645\u0641\u062A\u06CC",
|
|
313
|
+
"\u0639\u0644\u0627\u0645\u06C1",
|
|
314
|
+
"\u0628\u06CC\u06AF\u0645",
|
|
315
|
+
"\u0648\u063A\u06CC\u0631\u06C1",
|
|
316
|
+
"\u0631\u062D\u0645\u062A\u06C1",
|
|
317
|
+
"\u0631\u0636\u06CC",
|
|
318
|
+
"\u062A\u0639\u0627\u0644\u06CC",
|
|
319
|
+
"\u062A\u0639\u0627\u0644\u06CC\u0670",
|
|
320
|
+
"\u0639\u0644\u06CC\u06C1",
|
|
321
|
+
"\u0627\u0644\u0633\u0644\u0627\u0645",
|
|
322
|
+
"\u0635\u0644\u06CC",
|
|
323
|
+
"\u0648\u0633\u0644\u0645"
|
|
324
|
+
];
|
|
325
|
+
var ABBREV_PATTERN = new RegExp(`(?:${ABBREVIATIONS.join("|")})[.\u06D4]`, "gu");
|
|
326
|
+
function countSentences(input, options) {
|
|
327
|
+
return splitSentences(input, options).length;
|
|
303
328
|
}
|
|
304
|
-
function splitSentences(input) {
|
|
329
|
+
function splitSentences(input, options = {}) {
|
|
305
330
|
if (!input) return [];
|
|
306
|
-
|
|
331
|
+
const { preserveTerminators = false } = options;
|
|
332
|
+
const PROTECTED_DOT = "\uE000";
|
|
333
|
+
const PROTECTED_URDU_FULL_STOP = "\uE001";
|
|
334
|
+
let sanitized = input.replace(/(\d)\.(\d)/gu, `$1${PROTECTED_DOT}$2`).replace(ABBREV_PATTERN, (match) => {
|
|
335
|
+
return match.replace(/\./g, PROTECTED_DOT).replace(/۔/g, PROTECTED_URDU_FULL_STOP);
|
|
336
|
+
});
|
|
337
|
+
if (preserveTerminators) {
|
|
338
|
+
const matches = sanitized.match(/[^۔؟?!.…]+[۔؟?!.…]+|[^۔؟?!.…]+$/gu) ?? [];
|
|
339
|
+
return matches.map(
|
|
340
|
+
(s) => s.replace(new RegExp(PROTECTED_DOT, "gu"), ".").replace(new RegExp(PROTECTED_URDU_FULL_STOP, "gu"), "\u06D4").trim()
|
|
341
|
+
).filter((s) => s.length > 0);
|
|
342
|
+
}
|
|
343
|
+
return sanitized.split(SENTENCE_SPLIT_RE).map(
|
|
344
|
+
(s) => s.replace(new RegExp(PROTECTED_DOT, "gu"), ".").replace(new RegExp(PROTECTED_URDU_FULL_STOP, "gu"), "\u06D4").trim()
|
|
345
|
+
).filter((s) => s.length > 0);
|
|
307
346
|
}
|
|
308
347
|
function analyzeUrdu(input) {
|
|
309
348
|
const text = input ?? "";
|
|
@@ -326,6 +365,264 @@ function analyzeUrdu(input) {
|
|
|
326
365
|
};
|
|
327
366
|
}
|
|
328
367
|
|
|
368
|
+
// src/stopwords.ts
|
|
369
|
+
var URDU_STOP_WORDS = /* @__PURE__ */ new Set([
|
|
370
|
+
// Tense auxiliaries & copulas
|
|
371
|
+
"\u06C1\u06D2",
|
|
372
|
+
"\u06C1\u06CC\u06BA",
|
|
373
|
+
"\u06C1\u0648\u06BA",
|
|
374
|
+
"\u06C1\u0648",
|
|
375
|
+
"\u062A\u06BE\u0627",
|
|
376
|
+
"\u062A\u06BE\u06CC",
|
|
377
|
+
"\u062A\u06BE\u06D2",
|
|
378
|
+
"\u062A\u06BE\u06CC\u06BA",
|
|
379
|
+
"\u06C1\u0648\u06AF\u0627",
|
|
380
|
+
"\u06C1\u0648\u06AF\u06CC",
|
|
381
|
+
"\u06C1\u0648\u0646\u06AF\u06D2",
|
|
382
|
+
"\u06C1\u0648\u06BA\u06AF\u06D2",
|
|
383
|
+
"\u06C1\u0648\u0646\u0627",
|
|
384
|
+
"\u06C1\u0648\u0646\u06D2",
|
|
385
|
+
"\u06C1\u0648\u0627",
|
|
386
|
+
"\u06C1\u0648\u0626\u06CC",
|
|
387
|
+
"\u06C1\u0648\u0626\u06D2",
|
|
388
|
+
// Postpositions, relations & prepositions
|
|
389
|
+
"\u06A9\u0627",
|
|
390
|
+
"\u06A9\u06CC",
|
|
391
|
+
"\u06A9\u06D2",
|
|
392
|
+
"\u06A9\u0648",
|
|
393
|
+
"\u0646\u06D2",
|
|
394
|
+
"\u0633\u06D2",
|
|
395
|
+
"\u067E\u0631",
|
|
396
|
+
"\u062A\u06A9",
|
|
397
|
+
"\u0645\u06CC\u06BA",
|
|
398
|
+
"\u0644\u06CC\u06D2",
|
|
399
|
+
"\u0633\u0627\u062A\u06BE",
|
|
400
|
+
"\u0628\u063A\u06CC\u0631",
|
|
401
|
+
"\u0637\u0631\u062D",
|
|
402
|
+
"\u0637\u0631\u0641",
|
|
403
|
+
"\u0628\u0627\u0631\u06D2",
|
|
404
|
+
"\u0628\u0639\u062F",
|
|
405
|
+
"\u067E\u06C1\u0644\u06D2",
|
|
406
|
+
"\u062F\u0648\u0631\u0627\u0646",
|
|
407
|
+
"\u062F\u0631\u0645\u06CC\u0627\u0646",
|
|
408
|
+
"\u0639\u0644\u0627\u0648\u06C1",
|
|
409
|
+
"\u0633\u0648\u0627\u0626\u06D2",
|
|
410
|
+
"\u0633\u0648\u0627",
|
|
411
|
+
"\u0645\u0637\u0627\u0628\u0642",
|
|
412
|
+
"\u0628\u0627\u0639\u062B",
|
|
413
|
+
"\u0630\u0631\u06CC\u0639\u06D2",
|
|
414
|
+
"\u062A\u062D\u062A",
|
|
415
|
+
"\u0627\u0648\u067E\u0631",
|
|
416
|
+
"\u0646\u06CC\u0686\u06D2",
|
|
417
|
+
"\u0622\u06AF\u06D2",
|
|
418
|
+
"\u067E\u06CC\u0686\u06BE\u06D2",
|
|
419
|
+
"\u0627\u0646\u062F\u0631",
|
|
420
|
+
"\u0628\u0627\u06C1\u0631",
|
|
421
|
+
"\u067E\u0627\u0633",
|
|
422
|
+
"\u0642\u0631\u06CC\u0628",
|
|
423
|
+
"\u0646\u0632\u062F\u06CC\u06A9",
|
|
424
|
+
// Conjunctions & discourse markers
|
|
425
|
+
"\u0627\u0648\u0631",
|
|
426
|
+
"\u06CC\u0627",
|
|
427
|
+
"\u0627\u06AF\u0631",
|
|
428
|
+
"\u0644\u06CC\u06A9\u0646",
|
|
429
|
+
"\u0645\u06AF\u0631",
|
|
430
|
+
"\u062A\u0648",
|
|
431
|
+
"\u0628\u06BE\u06CC",
|
|
432
|
+
"\u06C1\u06CC",
|
|
433
|
+
"\u0646\u06C1",
|
|
434
|
+
"\u0646\u06C1\u06CC\u06BA",
|
|
435
|
+
"\u0645\u062A",
|
|
436
|
+
"\u0646\u0627",
|
|
437
|
+
"\u0628\u0644\u06A9\u06C1",
|
|
438
|
+
"\u062D\u0627\u0644\u0627\u0646\u06A9\u06C1",
|
|
439
|
+
"\u0686\u0648\u0646\u06A9\u06C1",
|
|
440
|
+
"\u06A9\u06CC\u0648\u0646\u06A9\u06C1",
|
|
441
|
+
"\u062A\u0627\u06A9\u06C1",
|
|
442
|
+
"\u0648\u0631\u0646\u06C1",
|
|
443
|
+
"\u0627\u0644\u0628\u062A\u06C1",
|
|
444
|
+
"\u0645\u062B\u0644\u0627\u064B",
|
|
445
|
+
"\u06CC\u0639\u0646\u06CC",
|
|
446
|
+
"\u062E\u0635\u0648\u0635\u0627\u064B",
|
|
447
|
+
"\u0639\u0645\u0648\u0645\u0627\u064B",
|
|
448
|
+
"\u0686\u0646\u0627\u0646\u0686\u06C1",
|
|
449
|
+
"\u062E\u0648\u0627\u06C1",
|
|
450
|
+
"\u0686\u0627\u06C1\u06D2",
|
|
451
|
+
"\u06AF\u0648\u06CC\u0627",
|
|
452
|
+
"\u062A\u0627\u06C1\u0645",
|
|
453
|
+
"\u0646\u06CC\u0632",
|
|
454
|
+
"\u062D\u062A\u06CC",
|
|
455
|
+
"\u0648\u063A\u06CC\u0631\u06C1",
|
|
456
|
+
"\u0635\u0631\u0641",
|
|
457
|
+
"\u0645\u062D\u0636",
|
|
458
|
+
"\u0641\u0642\u0637",
|
|
459
|
+
// Pronouns, determiners & question words
|
|
460
|
+
"\u06C1\u0645",
|
|
461
|
+
"\u062A\u0645",
|
|
462
|
+
"\u0622\u067E",
|
|
463
|
+
"\u0648\u06C1",
|
|
464
|
+
"\u06CC\u06C1",
|
|
465
|
+
"\u0645\u06CC\u0631\u0627",
|
|
466
|
+
"\u0645\u06CC\u0631\u06CC",
|
|
467
|
+
"\u0645\u06CC\u0631\u06D2",
|
|
468
|
+
"\u06C1\u0645\u0627\u0631\u0627",
|
|
469
|
+
"\u06C1\u0645\u0627\u0631\u06CC",
|
|
470
|
+
"\u06C1\u0645\u0627\u0631\u06D2",
|
|
471
|
+
"\u062A\u0645\u06C1\u0627\u0631\u0627",
|
|
472
|
+
"\u062A\u0645\u06C1\u0627\u0631\u06CC",
|
|
473
|
+
"\u062A\u0645\u06C1\u0627\u0631\u06D2",
|
|
474
|
+
"\u0622\u067E\u06A9\u0627",
|
|
475
|
+
"\u0622\u067E\u06A9\u06CC",
|
|
476
|
+
"\u0622\u067E\u06A9\u06D2",
|
|
477
|
+
"\u0627\u0633",
|
|
478
|
+
"\u0627\u0633\u06A9\u0627",
|
|
479
|
+
"\u0627\u0633\u06A9\u06CC",
|
|
480
|
+
"\u0627\u0633\u06A9\u06D2",
|
|
481
|
+
"\u0627\u0633\u06D2",
|
|
482
|
+
"\u0627\u0646",
|
|
483
|
+
"\u0627\u0646\u06A9\u0627",
|
|
484
|
+
"\u0627\u0646\u06A9\u06CC",
|
|
485
|
+
"\u0627\u0646\u06A9\u06D2",
|
|
486
|
+
"\u0627\u0646\u06C1\u06CC\u06BA",
|
|
487
|
+
"\u0645\u062C\u06BE\u06D2",
|
|
488
|
+
"\u06C1\u0645\u06CC\u06BA",
|
|
489
|
+
"\u062A\u0645\u06C1\u06CC\u06BA",
|
|
490
|
+
"\u062C\u0633",
|
|
491
|
+
"\u062C\u0633\u06A9\u0627",
|
|
492
|
+
"\u062C\u0633\u06A9\u06CC",
|
|
493
|
+
"\u062C\u0633\u06A9\u06D2",
|
|
494
|
+
"\u062C\u0633\u06D2",
|
|
495
|
+
"\u062C\u0646",
|
|
496
|
+
"\u062C\u0646\u06A9\u0627",
|
|
497
|
+
"\u062C\u0646\u06A9\u06CC",
|
|
498
|
+
"\u062C\u0646\u06A9\u06D2",
|
|
499
|
+
"\u062C\u0646\u06C1\u0648\u06BA",
|
|
500
|
+
"\u0627\u0646\u06C1\u0648\u06BA",
|
|
501
|
+
"\u06A9\u0633",
|
|
502
|
+
"\u06A9\u0633\u06A9\u0627",
|
|
503
|
+
"\u06A9\u0633\u06A9\u06CC",
|
|
504
|
+
"\u06A9\u0633\u06A9\u06D2",
|
|
505
|
+
"\u06A9\u0633\u06D2",
|
|
506
|
+
"\u06A9\u0633\u06A9\u0648",
|
|
507
|
+
"\u06A9\u0648\u0646",
|
|
508
|
+
"\u06A9\u06CC\u0627",
|
|
509
|
+
"\u06A9\u06C1\u0627\u06BA",
|
|
510
|
+
"\u06A9\u062F\u06BE\u0631",
|
|
511
|
+
"\u0627\u062F\u06BE\u0631",
|
|
512
|
+
"\u062C\u062F\u06BE\u0631",
|
|
513
|
+
"\u06A9\u0628",
|
|
514
|
+
"\u06A9\u06CC\u0648\u06BA",
|
|
515
|
+
"\u06A9\u06CC\u0633\u06D2",
|
|
516
|
+
"\u06A9\u06CC\u0633\u0627",
|
|
517
|
+
"\u06A9\u06CC\u0633\u06CC",
|
|
518
|
+
"\u06A9\u062A\u0646\u0627",
|
|
519
|
+
"\u06A9\u062A\u0646\u06CC",
|
|
520
|
+
"\u06A9\u062A\u0646\u06D2",
|
|
521
|
+
"\u062C\u0648",
|
|
522
|
+
"\u0633\u0628",
|
|
523
|
+
"\u0633\u0628\u06BE\u06CC",
|
|
524
|
+
"\u06A9\u0648\u0626\u06CC",
|
|
525
|
+
"\u06A9\u0686\u06BE",
|
|
526
|
+
"\u06C1\u0631",
|
|
527
|
+
"\u0627\u06CC\u06A9",
|
|
528
|
+
"\u0627\u067E\u0646\u0627",
|
|
529
|
+
"\u0627\u067E\u0646\u06CC",
|
|
530
|
+
"\u0627\u067E\u0646\u06D2",
|
|
531
|
+
"\u062E\u0648\u062F",
|
|
532
|
+
// Common light/auxiliary verb forms
|
|
533
|
+
"\u06A9\u0631\u0646\u0627",
|
|
534
|
+
"\u06A9\u0631\u062A\u0627",
|
|
535
|
+
"\u06A9\u0631\u062A\u06CC",
|
|
536
|
+
"\u06A9\u0631\u062A\u06D2",
|
|
537
|
+
"\u06A9\u0631\u06CC\u06BA",
|
|
538
|
+
"\u06A9\u0631\u0648",
|
|
539
|
+
"\u06A9\u0631",
|
|
540
|
+
"\u06A9\u0631\u0646\u06D2",
|
|
541
|
+
"\u06A9\u06CC\u06D2",
|
|
542
|
+
"\u062C\u0627\u0646\u0627",
|
|
543
|
+
"\u062C\u0627\u062A\u0627",
|
|
544
|
+
"\u062C\u0627\u062A\u06CC",
|
|
545
|
+
"\u062C\u0627\u062A\u06D2",
|
|
546
|
+
"\u062C\u0627\u0626\u06D2",
|
|
547
|
+
"\u062C\u0627\u0626\u06CC\u06BA",
|
|
548
|
+
"\u062C\u0627\u0624",
|
|
549
|
+
"\u062C\u0627",
|
|
550
|
+
"\u062C\u0627\u0646\u06D2",
|
|
551
|
+
"\u06AF\u06CC\u0627",
|
|
552
|
+
"\u06AF\u0626\u06CC",
|
|
553
|
+
"\u06AF\u0626\u06D2",
|
|
554
|
+
"\u0622\u0646\u0627",
|
|
555
|
+
"\u0622\u062A\u0627",
|
|
556
|
+
"\u0622\u062A\u06CC",
|
|
557
|
+
"\u0622\u062A\u06D2",
|
|
558
|
+
"\u0622\u0626\u06D2",
|
|
559
|
+
"\u0622\u0626\u06CC\u06BA",
|
|
560
|
+
"\u0622\u06CC\u0627",
|
|
561
|
+
"\u0622\u0626\u06CC",
|
|
562
|
+
"\u0622\u0646\u06D2",
|
|
563
|
+
"\u0622\u0624",
|
|
564
|
+
"\u0622",
|
|
565
|
+
"\u062F\u06CC\u0646\u0627",
|
|
566
|
+
"\u062F\u06CC\u062A\u0627",
|
|
567
|
+
"\u062F\u06CC\u062A\u06CC",
|
|
568
|
+
"\u062F\u06CC\u062A\u06D2",
|
|
569
|
+
"\u062F\u06CC\u0627",
|
|
570
|
+
"\u062F\u06CC\u06D2",
|
|
571
|
+
"\u062F\u06D2",
|
|
572
|
+
"\u062F\u0648",
|
|
573
|
+
"\u062F\u06CC\u06BA",
|
|
574
|
+
"\u062F\u06CC\u0646\u06D2",
|
|
575
|
+
"\u0644\u06CC\u0646\u0627",
|
|
576
|
+
"\u0644\u06CC\u062A\u0627",
|
|
577
|
+
"\u0644\u06CC\u062A\u06CC",
|
|
578
|
+
"\u0644\u06CC\u062A\u06D2",
|
|
579
|
+
"\u0644\u06CC\u0627",
|
|
580
|
+
"\u0644\u06D2",
|
|
581
|
+
"\u0644\u0648",
|
|
582
|
+
"\u0644\u06CC\u06BA",
|
|
583
|
+
"\u0644\u06CC\u0646\u06D2",
|
|
584
|
+
"\u0631\u06C1\u0646\u0627",
|
|
585
|
+
"\u0631\u06C1\u0627",
|
|
586
|
+
"\u0631\u06C1\u06CC",
|
|
587
|
+
"\u0631\u06C1\u06D2",
|
|
588
|
+
"\u0631\u06C1\u062A\u0627",
|
|
589
|
+
"\u0631\u06C1\u062A\u06CC",
|
|
590
|
+
"\u0631\u06C1\u062A\u06D2",
|
|
591
|
+
"\u0631\u06C1\u0646\u06D2",
|
|
592
|
+
"\u0633\u06A9\u0646\u0627",
|
|
593
|
+
"\u0633\u06A9\u062A\u0627",
|
|
594
|
+
"\u0633\u06A9\u062A\u06CC",
|
|
595
|
+
"\u0633\u06A9\u062A\u06D2",
|
|
596
|
+
"\u0648\u0627\u0644\u0627",
|
|
597
|
+
"\u0648\u0627\u0644\u06CC",
|
|
598
|
+
"\u0648\u0627\u0644\u06D2"
|
|
599
|
+
]);
|
|
600
|
+
function toStopWordSet(custom) {
|
|
601
|
+
if (!custom) return URDU_STOP_WORDS;
|
|
602
|
+
if (custom instanceof Set) return custom;
|
|
603
|
+
return new Set(custom.map((w) => normalizeUrdu(w.trim())));
|
|
604
|
+
}
|
|
605
|
+
function isStopWord(word, customStopWords) {
|
|
606
|
+
if (!word) return false;
|
|
607
|
+
const normalized = normalizeUrdu(word.trim());
|
|
608
|
+
const stopSet = toStopWordSet(customStopWords);
|
|
609
|
+
return stopSet.has(normalized);
|
|
610
|
+
}
|
|
611
|
+
function filterStopWords(words, customStopWords) {
|
|
612
|
+
if (!words || words.length === 0) return [];
|
|
613
|
+
const stopSet = toStopWordSet(customStopWords);
|
|
614
|
+
return words.filter((w) => {
|
|
615
|
+
const normalized = normalizeUrdu(w.trim());
|
|
616
|
+
return normalized.length > 0 && !stopSet.has(normalized);
|
|
617
|
+
});
|
|
618
|
+
}
|
|
619
|
+
function removeStopWords(text, customStopWords) {
|
|
620
|
+
if (!text) return "";
|
|
621
|
+
const words = splitWords(text);
|
|
622
|
+
const filtered = filterStopWords(words, customStopWords);
|
|
623
|
+
return filtered.join(" ");
|
|
624
|
+
}
|
|
625
|
+
|
|
329
626
|
// src/collate.ts
|
|
330
627
|
var ALPHABET = [
|
|
331
628
|
"\u0627",
|
|
@@ -2997,6 +3294,6 @@ function urduSlug(input, options = {}) {
|
|
|
2997
3294
|
return slug;
|
|
2998
3295
|
}
|
|
2999
3296
|
|
|
3000
|
-
export { analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
|
3297
|
+
export { URDU_STOP_WORDS, analyzeUrdu, compareUrdu, convertNumbers, countSentences, countWords, editDistance, filterStopWords, foldUrdu, hasUrduSpecificLetters, highlightUrdu, isStopWord, isUrdu, normalizeUrdu, numberToUrduWords, parseUrduNumber, removeDiacritics, removeStopWords, romanToUrdu, romanize, searchUrdu, searchUrduRanked, sortUrdu, splitSentences, splitWords, toArabicIndicDigits, toEnglishDigits, toUrduDigits, urduRatio, urduSlug };
|
|
3001
3298
|
//# sourceMappingURL=index.js.map
|
|
3002
3299
|
//# sourceMappingURL=index.js.map
|