urdu-text-utils 0.1.3 → 0.1.4

package/CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.1.4
+
+- Add Urdu stop words module (`URDU_STOP_WORDS`, `isStopWord`, `filterStopWords`, `removeStopWords`) with a curated set of 130+ canonical functional words.
+- Enhance sentence segmentation in `splitSentences` and `countSentences` with abbreviation & honorific protection (e.g. `ڈاکٹر.`, `پروفیسر.`, `صاحب.`), numeric decimal protection, and optional `preserveTerminators` option.
+
 ## 0.1.3
 
 - Significantly expand transliteration dictionary coverage across pronouns & oblique forms (`مجھے`, `ہمیں`, `انہیں`, `اسے`, `جسے`), everyday verbs and inflections, calendar & weekdays, numbers & ordinals, food, household objects, health, geography, and common Pakistani names.

package/README.md

@@ -76,14 +76,30 @@ hasUrduSpecificLetters("لڑکی"); // true  — ڑ does not exist in Arabic
 hasUrduSpecificLetters("كتاب مدرسة"); // false
 ```
 
-## Word and sentence counting
+## Word and sentence tokenization
 
 ```ts
 countWords("پاکستان ایک خوبصورت ملک ہے"); // 5
 countWords("آپ کیسے ہیں؟"); // 3 — attached punctuation is not a word
 countSentences("یہ پہلا جملہ ہے۔ یہ دوسرا ہے۔"); // 2
 splitWords(text); // string[]
-splitSentences(text); // string[]
+splitSentences(text); // string[] — protects abbreviations like ڈاکٹر. and decimals
+splitSentences(text, { preserveTerminators: true }); // preserves ending punctuation
+```
+
+## Stop words
+
+```ts
+import { isStopWord, filterStopWords, removeStopWords, URDU_STOP_WORDS } from "urdu-text-utils";
+
+isStopWord("اور"); // true
+isStopWord("کتاب"); // false
+
+filterStopWords(["یہ", "ایک", "بہترین", "کتاب", "ہے"]);
+// ["بہترین", "کتاب"]
+
+removeStopWords("پاکستان ایک خوبصورت ملک ہے");
+// "پاکستان خوبصورت ملک"
 ```
 
 ## Urdu numbers

package/dist/index.cjs

@@ -300,12 +300,51 @@ function splitWords(input) {
   if (!input) return [];
   return input.split(WORD_SPLIT_RE).map((w) => w.trim()).filter((w) => w.length > 0);
 }
-function countSentences(input) {
-  return splitSentences(input).length;
+var ABBREVIATIONS = [
+  "\u0688\u0627\u06A9\u0679\u0631",
+  "\u067E\u0631\u0648\u0641\u06CC\u0633\u0631",
+  "\u0627\u0646\u062C\u06CC\u0646\u0626\u0631",
+  "\u0627\u06CC\u0688\u0648\u0648\u06A9\u06CC\u0679",
+  "\u062C\u0646\u0627\u0628",
+  "\u0635\u0627\u062D\u0628",
+  "\u0635\u0627\u062D\u0628\u06C1",
+  "\u0645\u062D\u062A\u0631\u0645",
+  "\u0645\u062D\u062A\u0631\u0645\u06C1",
+  "\u0645\u0648\u0644\u0627\u0646\u0627",
+  "\u0645\u0641\u062A\u06CC",
+  "\u0639\u0644\u0627\u0645\u06C1",
+  "\u0628\u06CC\u06AF\u0645",
+  "\u0648\u063A\u06CC\u0631\u06C1",
+  "\u0631\u062D\u0645\u062A\u06C1",
+  "\u0631\u0636\u06CC",
+  "\u062A\u0639\u0627\u0644\u06CC",
+  "\u062A\u0639\u0627\u0644\u06CC\u0670",
+  "\u0639\u0644\u06CC\u06C1",
+  "\u0627\u0644\u0633\u0644\u0627\u0645",
+  "\u0635\u0644\u06CC",
+  "\u0648\u0633\u0644\u0645"
+];
+var ABBREV_PATTERN = new RegExp(`(?:${ABBREVIATIONS.join("|")})[.\u06D4]`, "gu");
+function countSentences(input, options) {
+  return splitSentences(input, options).length;
 }
-function splitSentences(input) {
+function splitSentences(input, options = {}) {
   if (!input) return [];
-  return input.split(SENTENCE_SPLIT_RE).map((s) => s.trim()).filter((s) => s.length > 0);
+  const { preserveTerminators = false } = options;
+  const PROTECTED_DOT = "\uE000";
+  const PROTECTED_URDU_FULL_STOP = "\uE001";
+  let sanitized = input.replace(/(\d)\.(\d)/gu, `$1${PROTECTED_DOT}$2`).replace(ABBREV_PATTERN, (match) => {
+    return match.replace(/\./g, PROTECTED_DOT).replace(/۔/g, PROTECTED_URDU_FULL_STOP);
+  });
+  if (preserveTerminators) {
+    const matches = sanitized.match(/[^۔؟?!.…]+[۔؟?!.…]+|[^۔؟?!.…]+$/gu) ?? [];
+    return matches.map(
+      (s) => s.replace(new RegExp(PROTECTED_DOT, "gu"), ".").replace(new RegExp(PROTECTED_URDU_FULL_STOP, "gu"), "\u06D4").trim()
+    ).filter((s) => s.length > 0);
+  }
+  return sanitized.split(SENTENCE_SPLIT_RE).map(
+    (s) => s.replace(new RegExp(PROTECTED_DOT, "gu"), ".").replace(new RegExp(PROTECTED_URDU_FULL_STOP, "gu"), "\u06D4").trim()
+  ).filter((s) => s.length > 0);
 }
 function analyzeUrdu(input) {
   const text = input ?? "";
@@ -328,6 +367,264 @@ function analyzeUrdu(input) {
   };
 }
 
+// src/stopwords.ts
+var URDU_STOP_WORDS = /* @__PURE__ */ new Set([
+  // Tense auxiliaries & copulas
+  "\u06C1\u06D2",
+  "\u06C1\u06CC\u06BA",
+  "\u06C1\u0648\u06BA",
+  "\u06C1\u0648",
+  "\u062A\u06BE\u0627",
+  "\u062A\u06BE\u06CC",
+  "\u062A\u06BE\u06D2",
+  "\u062A\u06BE\u06CC\u06BA",
+  "\u06C1\u0648\u06AF\u0627",
+  "\u06C1\u0648\u06AF\u06CC",
+  "\u06C1\u0648\u0646\u06AF\u06D2",
+  "\u06C1\u0648\u06BA\u06AF\u06D2",
+  "\u06C1\u0648\u0646\u0627",
+  "\u06C1\u0648\u0646\u06D2",
+  "\u06C1\u0648\u0627",
+  "\u06C1\u0648\u0626\u06CC",
+  "\u06C1\u0648\u0626\u06D2",
+  // Postpositions, relations & prepositions
+  "\u06A9\u0627",
+  "\u06A9\u06CC",
+  "\u06A9\u06D2",
+  "\u06A9\u0648",
+  "\u0646\u06D2",
+  "\u0633\u06D2",
+  "\u067E\u0631",
+  "\u062A\u06A9",
+  "\u0645\u06CC\u06BA",
+  "\u0644\u06CC\u06D2",
+  "\u0633\u0627\u062A\u06BE",
+  "\u0628\u063A\u06CC\u0631",
+  "\u0637\u0631\u062D",
+  "\u0637\u0631\u0641",
+  "\u0628\u0627\u0631\u06D2",
+  "\u0628\u0639\u062F",
+  "\u067E\u06C1\u0644\u06D2",
+  "\u062F\u0648\u0631\u0627\u0646",
+  "\u062F\u0631\u0645\u06CC\u0627\u0646",
+  "\u0639\u0644\u0627\u0648\u06C1",
+  "\u0633\u0648\u0627\u0626\u06D2",
+  "\u0633\u0648\u0627",
+  "\u0645\u0637\u0627\u0628\u0642",
+  "\u0628\u0627\u0639\u062B",
+  "\u0630\u0631\u06CC\u0639\u06D2",
+  "\u062A\u062D\u062A",
+  "\u0627\u0648\u067E\u0631",
+  "\u0646\u06CC\u0686\u06D2",
+  "\u0622\u06AF\u06D2",
+  "\u067E\u06CC\u0686\u06BE\u06D2",
+  "\u0627\u0646\u062F\u0631",
+  "\u0628\u0627\u06C1\u0631",
+  "\u067E\u0627\u0633",
+  "\u0642\u0631\u06CC\u0628",
+  "\u0646\u0632\u062F\u06CC\u06A9",
+  // Conjunctions & discourse markers
+  "\u0627\u0648\u0631",
+  "\u06CC\u0627",
+  "\u0627\u06AF\u0631",
+  "\u0644\u06CC\u06A9\u0646",
+  "\u0645\u06AF\u0631",
+  "\u062A\u0648",
+  "\u0628\u06BE\u06CC",
+  "\u06C1\u06CC",
+  "\u0646\u06C1",
+  "\u0646\u06C1\u06CC\u06BA",
+  "\u0645\u062A",
+  "\u0646\u0627",
+  "\u0628\u0644\u06A9\u06C1",
+  "\u062D\u0627\u0644\u0627\u0646\u06A9\u06C1",
+  "\u0686\u0648\u0646\u06A9\u06C1",
+  "\u06A9\u06CC\u0648\u0646\u06A9\u06C1",
+  "\u062A\u0627\u06A9\u06C1",
+  "\u0648\u0631\u0646\u06C1",
+  "\u0627\u0644\u0628\u062A\u06C1",
+  "\u0645\u062B\u0644\u0627\u064B",
+  "\u06CC\u0639\u0646\u06CC",
+  "\u062E\u0635\u0648\u0635\u0627\u064B",
+  "\u0639\u0645\u0648\u0645\u0627\u064B",
+  "\u0686\u0646\u0627\u0646\u0686\u06C1",
+  "\u062E\u0648\u0627\u06C1",
+  "\u0686\u0627\u06C1\u06D2",
+  "\u06AF\u0648\u06CC\u0627",
+  "\u062A\u0627\u06C1\u0645",
+  "\u0646\u06CC\u0632",
+  "\u062D\u062A\u06CC",
+  "\u0648\u063A\u06CC\u0631\u06C1",
+  "\u0635\u0631\u0641",
+  "\u0645\u062D\u0636",
+  "\u0641\u0642\u0637",
+  // Pronouns, determiners & question words
+  "\u06C1\u0645",
+  "\u062A\u0645",
+  "\u0622\u067E",
+  "\u0648\u06C1",
+  "\u06CC\u06C1",
+  "\u0645\u06CC\u0631\u0627",
+  "\u0645\u06CC\u0631\u06CC",
+  "\u0645\u06CC\u0631\u06D2",
+  "\u06C1\u0645\u0627\u0631\u0627",
+  "\u06C1\u0645\u0627\u0631\u06CC",
+  "\u06C1\u0645\u0627\u0631\u06D2",
+  "\u062A\u0645\u06C1\u0627\u0631\u0627",
+  "\u062A\u0645\u06C1\u0627\u0631\u06CC",
+  "\u062A\u0645\u06C1\u0627\u0631\u06D2",
+  "\u0622\u067E\u06A9\u0627",
+  "\u0622\u067E\u06A9\u06CC",
+  "\u0622\u067E\u06A9\u06D2",
+  "\u0627\u0633",
+  "\u0627\u0633\u06A9\u0627",
+  "\u0627\u0633\u06A9\u06CC",
+  "\u0627\u0633\u06A9\u06D2",
+  "\u0627\u0633\u06D2",
+  "\u0627\u0646",
+  "\u0627\u0646\u06A9\u0627",
+  "\u0627\u0646\u06A9\u06CC",
+  "\u0627\u0646\u06A9\u06D2",
+  "\u0627\u0646\u06C1\u06CC\u06BA",
+  "\u0645\u062C\u06BE\u06D2",
+  "\u06C1\u0645\u06CC\u06BA",
+  "\u062A\u0645\u06C1\u06CC\u06BA",
+  "\u062C\u0633",
+  "\u062C\u0633\u06A9\u0627",
+  "\u062C\u0633\u06A9\u06CC",
+  "\u062C\u0633\u06A9\u06D2",
+  "\u062C\u0633\u06D2",
+  "\u062C\u0646",
+  "\u062C\u0646\u06A9\u0627",
+  "\u062C\u0646\u06A9\u06CC",
+  "\u062C\u0646\u06A9\u06D2",
+  "\u062C\u0646\u06C1\u0648\u06BA",
+  "\u0627\u0646\u06C1\u0648\u06BA",
+  "\u06A9\u0633",
+  "\u06A9\u0633\u06A9\u0627",
+  "\u06A9\u0633\u06A9\u06CC",
+  "\u06A9\u0633\u06A9\u06D2",
+  "\u06A9\u0633\u06D2",
+  "\u06A9\u0633\u06A9\u0648",
+  "\u06A9\u0648\u0646",
+  "\u06A9\u06CC\u0627",
+  "\u06A9\u06C1\u0627\u06BA",
+  "\u06A9\u062F\u06BE\u0631",
+  "\u0627\u062F\u06BE\u0631",
+  "\u062C\u062F\u06BE\u0631",
+  "\u06A9\u0628",
+  "\u06A9\u06CC\u0648\u06BA",
+  "\u06A9\u06CC\u0633\u06D2",
+  "\u06A9\u06CC\u0633\u0627",
+  "\u06A9\u06CC\u0633\u06CC",
+  "\u06A9\u062A\u0646\u0627",
+  "\u06A9\u062A\u0646\u06CC",
+  "\u06A9\u062A\u0646\u06D2",
+  "\u062C\u0648",
+  "\u0633\u0628",
+  "\u0633\u0628\u06BE\u06CC",
+  "\u06A9\u0648\u0626\u06CC",
+  "\u06A9\u0686\u06BE",
+  "\u06C1\u0631",
+  "\u0627\u06CC\u06A9",
+  "\u0627\u067E\u0646\u0627",
+  "\u0627\u067E\u0646\u06CC",
+  "\u0627\u067E\u0646\u06D2",
+  "\u062E\u0648\u062F",
+  // Common light/auxiliary verb forms
+  "\u06A9\u0631\u0646\u0627",
+  "\u06A9\u0631\u062A\u0627",
+  "\u06A9\u0631\u062A\u06CC",
+  "\u06A9\u0631\u062A\u06D2",
+  "\u06A9\u0631\u06CC\u06BA",
+  "\u06A9\u0631\u0648",
+  "\u06A9\u0631",
+  "\u06A9\u0631\u0646\u06D2",
+  "\u06A9\u06CC\u06D2",
+  "\u062C\u0627\u0646\u0627",
+  "\u062C\u0627\u062A\u0627",
+  "\u062C\u0627\u062A\u06CC",
+  "\u062C\u0627\u062A\u06D2",
+  "\u062C\u0627\u0626\u06D2",
+  "\u062C\u0627\u0626\u06CC\u06BA",
+  "\u062C\u0627\u0624",
+  "\u062C\u0627",
+  "\u062C\u0627\u0646\u06D2",
+  "\u06AF\u06CC\u0627",
+  "\u06AF\u0626\u06CC",
+  "\u06AF\u0626\u06D2",
+  "\u0622\u0646\u0627",
+  "\u0622\u062A\u0627",
+  "\u0622\u062A\u06CC",
+  "\u0622\u062A\u06D2",
+  "\u0622\u0626\u06D2",
+  "\u0622\u0626\u06CC\u06BA",
+  "\u0622\u06CC\u0627",
+  "\u0622\u0626\u06CC",
+  "\u0622\u0646\u06D2",
+  "\u0622\u0624",
+  "\u0622",
+  "\u062F\u06CC\u0646\u0627",
+  "\u062F\u06CC\u062A\u0627",
+  "\u062F\u06CC\u062A\u06CC",
+  "\u062F\u06CC\u062A\u06D2",
+  "\u062F\u06CC\u0627",
+  "\u062F\u06CC\u06D2",
+  "\u062F\u06D2",
+  "\u062F\u0648",
+  "\u062F\u06CC\u06BA",
+  "\u062F\u06CC\u0646\u06D2",
+  "\u0644\u06CC\u0646\u0627",
+  "\u0644\u06CC\u062A\u0627",
+  "\u0644\u06CC\u062A\u06CC",
+  "\u0644\u06CC\u062A\u06D2",
+  "\u0644\u06CC\u0627",
+  "\u0644\u06D2",
+  "\u0644\u0648",
+  "\u0644\u06CC\u06BA",
+  "\u0644\u06CC\u0646\u06D2",
+  "\u0631\u06C1\u0646\u0627",
+  "\u0631\u06C1\u0627",
+  "\u0631\u06C1\u06CC",
+  "\u0631\u06C1\u06D2",
+  "\u0631\u06C1\u062A\u0627",
+  "\u0631\u06C1\u062A\u06CC",
+  "\u0631\u06C1\u062A\u06D2",
+  "\u0631\u06C1\u0646\u06D2",
+  "\u0633\u06A9\u0646\u0627",
+  "\u0633\u06A9\u062A\u0627",
+  "\u0633\u06A9\u062A\u06CC",
+  "\u0633\u06A9\u062A\u06D2",
+  "\u0648\u0627\u0644\u0627",
+  "\u0648\u0627\u0644\u06CC",
+  "\u0648\u0627\u0644\u06D2"
+]);
+function toStopWordSet(custom) {
+  if (!custom) return URDU_STOP_WORDS;
+  if (custom instanceof Set) return custom;
+  return new Set(custom.map((w) => normalizeUrdu(w.trim())));
+}
+function isStopWord(word, customStopWords) {
+  if (!word) return false;
+  const normalized = normalizeUrdu(word.trim());
+  const stopSet = toStopWordSet(customStopWords);
+  return stopSet.has(normalized);
+}
+function filterStopWords(words, customStopWords) {
+  if (!words || words.length === 0) return [];
+  const stopSet = toStopWordSet(customStopWords);
+  return words.filter((w) => {
+    const normalized = normalizeUrdu(w.trim());
+    return normalized.length > 0 && !stopSet.has(normalized);
+  });
+}
+function removeStopWords(text, customStopWords) {
+  if (!text) return "";
+  const words = splitWords(text);
+  const filtered = filterStopWords(words, customStopWords);
+  return filtered.join(" ");
+}
+
 // src/collate.ts
 var ALPHABET = [
   "\u0627",
@@ -2999,20 +3296,24 @@ function urduSlug(input, options = {}) {
   return slug;
 }
 
+exports.URDU_STOP_WORDS = URDU_STOP_WORDS;
 exports.analyzeUrdu = analyzeUrdu;
 exports.compareUrdu = compareUrdu;
 exports.convertNumbers = convertNumbers;
 exports.countSentences = countSentences;
 exports.countWords = countWords;
 exports.editDistance = editDistance;
+exports.filterStopWords = filterStopWords;
 exports.foldUrdu = foldUrdu;
 exports.hasUrduSpecificLetters = hasUrduSpecificLetters;
 exports.highlightUrdu = highlightUrdu;
+exports.isStopWord = isStopWord;
 exports.isUrdu = isUrdu;
 exports.normalizeUrdu = normalizeUrdu;
 exports.numberToUrduWords = numberToUrduWords;
 exports.parseUrduNumber = parseUrduNumber;
 exports.removeDiacritics = removeDiacritics;
+exports.removeStopWords = removeStopWords;
 exports.romanToUrdu = romanToUrdu;
 exports.romanize = romanize;
 exports.searchUrdu = searchUrdu;