urdu-text-utils 0.1.8 → 0.1.9

package/CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.1.9
+
+- Add rule-based Urdu Stemmer module (`stemUrdu`, `stemUrduText`, `getAffixes`).
+- Add morphological restorations for feminine plurals (*لڑکیاں* → *لڑکی*), hamza plurals (*دعاؤں* → *دعا*), and Arabic/sound plurals (*تعلیمات* → *تعلیم*).
+- Add canonical prefix and suffix stripping with short root protection and exception handling.
+- Add canonical constants `URDU_PREFIXES` and `URDU_SUFFIXES`.
+
 ## 0.1.8
 
 - Fix jsDelivr monthly hit badge in README to use shields.io SVG endpoint.

package/README.md

@@ -53,6 +53,8 @@ import {
   removeStopWords,
   formatUrduDate,
   timeAgoUrdu,
+  stemUrdu,
+  stemUrduText,
   isUrdu,
   countWords,
   splitSentences,
@@ -172,6 +174,31 @@ timeAgoUrdu(Date.now() - 2 * 86400 * 1000);// "پرسوں"
 timeAgoUrdu(Date.now() + 10 * 60 * 1000);  // "۱۰ منٹ بعد"
 ```
 
+## Urdu stemming
+
+Rule-based stemmer and affix stripper with morphological vowel/letter restorations:
+
+```ts
+import { stemUrdu, stemUrduText } from "urdu-text-utils";
+
+// Plurals & morphological restoration
+stemUrdu("کتابیں");   // "کتاب"
+stemUrdu("لڑکیاں");   // "لڑکی" (restores final ی)
+stemUrdu("کہانیاں");  // "کہانی"
+stemUrdu("دعاؤں");    // "دعا"
+stemUrdu("خوشبوئیں"); // "خوشبو"
+
+// Prefixes & derivational suffixes
+stemUrdu("بےوقوف");   // "وقوف"
+stemUrdu("نااہل");    // "اہل"
+stemUrdu("دکاندار");  // "دکان"
+stemUrdu("مددگار");   // "مدد"
+
+// Stems full text while preserving layout & punctuation
+stemUrduText("طلباء کتابیں پڑھتے ہیں اور کہانیاں سنتے ہیں۔");
+// "طلباء کتاب پڑھ ہیں اور کہانی سن ہیں۔"
+```
+
 ## Diacritics
 
 ```ts

package/dist/index.cjs

@@ -3703,9 +3703,249 @@ function timeAgoUrdu(date, relativeTo = /* @__PURE__ */ new Date(), options = {}
   return isPast ? `${count} \u0633\u0627\u0644 \u067E\u06C1\u0644\u06D2` : `${count} \u0633\u0627\u0644 \u0628\u0639\u062F`;
 }
 
+// src/stemmer.ts
+var URDU_PREFIXES = [
+  "\u063A\u06CC\u0631",
+  "\u062E\u0648\u062F",
+  "\u0627\u06C1\u0644",
+  "\u0628\u06D2",
+  "\u0646\u0627",
+  "\u0644\u0627",
+  "\u0628\u062F",
+  "\u06A9\u0645",
+  "\u0627\u0646",
+  "\u06C1\u0645",
+  "\u0628\u0627",
+  "\u067E\u064F\u0631"
+];
+var URDU_SUFFIXES = [
+  // Compound / multi-syllable suffixes
+  "\u06CC\u06BA\u06AF\u06D2",
+  "\u06CC\u0646\u06AF\u06D2",
+  "\u06D2\u06AF\u0627",
+  "\u06D2\u06AF\u06CC",
+  "\u06CC\u0646\u06AF\u06CC",
+  "\u062F\u0627\u0631\u06CC",
+  "\u06AF\u0627\u0631\u06CC",
+  "\u06A9\u0627\u0631\u06CC",
+  "\u0633\u0627\u0632\u06CC",
+  "\u0628\u0627\u0632\u06CC",
+  "\u0645\u0646\u062F\u06CC",
+  "\u062A\u0631\u06CC\u0646",
+  "\u0633\u062A\u0627\u0646",
+  "\u062E\u0627\u0646\u06C1",
+  "\u0646\u0627\u0645\u06C1",
+  "\u062C\u0627\u062A",
+  "\u06AF\u0627\u0646",
+  // Adjectival / nominal / agentive suffixes
+  "\u062F\u0627\u0631",
+  "\u06AF\u0627\u0631",
+  "\u06A9\u0627\u0631",
+  "\u0633\u0627\u0632",
+  "\u0628\u0627\u0632",
+  "\u0645\u0646\u062F",
+  "\u0646\u0627\u06A9",
+  "\u0648\u0627\u0631",
+  "\u062F\u0627\u0646",
+  "\u0632\u0627\u0631",
+  "\u06CC\u062A",
+  "\u067E\u0646",
+  "\u0627\u0626\u06CC",
+  "\u062A\u0631",
+  // Plural & verbal inflections
+  "\u06CC\u0627\u06BA",
+  "\u0626\u06CC\u06BA",
+  "\u0624\u06BA",
+  "\u06CC\u0648\u06BA",
+  "\u0648\u06BA",
+  "\u06CC\u06BA",
+  "\u0627\u062A",
+  "\u06C1\u0627",
+  "\u06CC\u0646",
+  "\u062A\u06CC\u06BA",
+  "\u062A\u0627",
+  "\u062A\u06CC",
+  "\u062A\u06D2",
+  "\u0646\u0627"
+];
+var PROTECTED_WORDS = /* @__PURE__ */ new Set([
+  "\u06C1\u0645",
+  "\u0646\u0627",
+  "\u0644\u0627",
+  "\u0628\u06D2",
+  "\u06CC\u0627",
+  "\u062A\u0648",
+  "\u067E\u0631",
+  "\u0633\u06D2",
+  "\u06A9\u0648",
+  "\u06A9\u0627",
+  "\u06A9\u06CC",
+  "\u06A9\u06D2",
+  "\u0645\u06CC\u06BA",
+  "\u0646\u06D2",
+  "\u06C1\u06D2",
+  "\u06C1\u06CC\u06BA",
+  "\u06C1\u0648",
+  "\u062A\u06BE\u0627",
+  "\u062A\u06BE\u06CC",
+  "\u062A\u06BE\u06D2",
+  "\u062A\u06BE\u06CC\u06BA",
+  "\u0628\u0627\u063A",
+  "\u0646\u0627\u0645",
+  "\u0633\u0641\u0631",
+  "\u06C1\u0648\u0627",
+  "\u062F\u0644",
+  "\u0633\u0631",
+  "\u0644\u0628",
+  "\u062F\u0633\u062A",
+  "\u06A9\u0627\u0631",
+  "\u0628\u0627\u0631",
+  "\u0628\u0627\u0644",
+  "\u0628\u0627\u0646",
+  "\u0634\u0627\u0645",
+  "\u06A9\u0627\u0645",
+  "\u062F\u0627\u0645",
+  "\u062C\u0627\u0645",
+  "\u0639\u0627\u0645",
+  "\u0631\u0627\u0645",
+  "\u06AF\u0627\u0645",
+  "\u062A\u0627\u0631",
+  "\u06CC\u0627\u0631",
+  "\u0645\u0627\u0631",
+  "\u0646\u0627\u0631",
+  "\u062F\u0627\u0631",
+  "\u06C1\u0627\u0631",
+  "\u0632\u0627\u0631",
+  "\u063A\u0627\u0631",
+  "\u0631\u0627\u062A",
+  "\u0628\u0627\u062A",
+  "\u0645\u0627\u062A",
+  "\u0630\u0627\u062A",
+  "\u06C1\u0627\u062A\u06BE",
+  "\u0633\u0627\u062A\u06BE",
+  "\u062F\u06CC\u0646",
+  "\u062A\u06CC\u0646",
+  "\u0686\u06CC\u0646",
+  "\u0628\u06CC\u0646",
+  "\u0632\u0645\u06CC\u0646",
+  "\u0627\u0646\u0633\u0627\u0646",
+  "\u0627\u062D\u0633\u0627\u0646",
+  "\u0627\u06CC\u0645\u0627\u0646",
+  "\u0627\u0633\u0644\u0627\u0645",
+  "\u0627\u0639\u0644\u0627\u0646",
+  "\u0627\u0645\u06A9\u0627\u0646",
+  "\u0627\u0631\u0645\u0627\u0646",
+  "\u0627\u0641\u063A\u0627\u0646",
+  "\u0627\u06CC\u0631\u0627\u0646",
+  "\u0639\u0645\u0631\u0627\u0646",
+  "\u0639\u062B\u0645\u0627\u0646",
+  "\u0633\u0644\u0645\u0627\u0646",
+  "\u0639\u0631\u0641\u0627\u0646",
+  "\u0642\u0631\u0622\u0646",
+  "\u0637\u0648\u0641\u0627\u0646",
+  "\u067E\u06C1\u0686\u0627\u0646",
+  "\u062F\u06A9\u0627\u0646",
+  "\u0645\u06A9\u0627\u0646",
+  "\u0627\u0633\u0645\u0627\u0646",
+  "\u0622\u0633\u0645\u0627\u0646",
+  "\u0645\u06C1\u0645\u0627\u0646",
+  "\u0646\u0642\u0635\u0627\u0646"
+]);
+function applySuffixStripping(word, suffixes, minLen) {
+  if (word.length <= minLen) return { stem: word };
+  if (word.endsWith("\u06CC\u0627\u06BA") && word.length - 3 >= minLen) {
+    const base = word.slice(0, -3);
+    return { stem: `${base}\u06CC`, suffix: "\u06CC\u0627\u06BA" };
+  }
+  if (word.endsWith("\u06CC\u0648\u06BA") && word.length - 3 >= minLen) {
+    const base = word.slice(0, -3);
+    return { stem: `${base}\u06CC`, suffix: "\u06CC\u0648\u06BA" };
+  }
+  if (word.endsWith("\u0626\u06CC\u06BA") && word.length - 3 >= minLen) {
+    const base = word.slice(0, -3);
+    return { stem: base, suffix: "\u0626\u06CC\u06BA" };
+  }
+  if (word.endsWith("\u0624\u06BA") && word.length - 2 >= minLen) {
+    const base = word.slice(0, -2);
+    return { stem: base, suffix: "\u0624\u06BA" };
+  }
+  for (const suf of suffixes) {
+    if (word.endsWith(suf)) {
+      const remainder = word.slice(0, -suf.length).replace(/[\s\u200C\u200D]+$/, "");
+      if (remainder.length >= minLen && !PROTECTED_WORDS.has(word)) {
+        return { stem: remainder, suffix: suf };
+      }
+    }
+  }
+  return { stem: word };
+}
+function applyPrefixStripping(word, prefixes, minLen) {
+  if (word.length <= minLen) return { stem: word };
+  for (const pref of prefixes) {
+    if (word.startsWith(pref)) {
+      const remainder = word.slice(pref.length).replace(/^[\s\u200C\u200D]+/, "");
+      if (remainder.length >= minLen && !PROTECTED_WORDS.has(word)) {
+        return { stem: remainder, prefix: pref };
+      }
+    }
+  }
+  return { stem: word };
+}
+function getAffixes(input, options = {}) {
+  if (!input || typeof input !== "string") {
+    return { stem: "" };
+  }
+  const clean = removeDiacritics(normalizeUrdu(input.trim()));
+  if (!clean) return { stem: "" };
+  if (options.exceptions && options.exceptions[clean]) {
+    return { stem: options.exceptions[clean] };
+  }
+  if (PROTECTED_WORDS.has(clean)) {
+    return { stem: clean };
+  }
+  const minLen = options.minStemLength ?? 2;
+  const doSuffixes = options.stripSuffixes ?? true;
+  const doPrefixes = options.stripPrefixes ?? true;
+  const suffixes = options.customSuffixes ?? URDU_SUFFIXES;
+  const prefixes = options.customPrefixes ?? URDU_PREFIXES;
+  let current = clean;
+  let detectedSuffix;
+  let detectedPrefix;
+  if (doSuffixes) {
+    const sRes = applySuffixStripping(current, suffixes, minLen);
+    current = sRes.stem;
+    detectedSuffix = sRes.suffix;
+  }
+  if (doPrefixes) {
+    const pRes = applyPrefixStripping(current, prefixes, minLen);
+    current = pRes.stem;
+    detectedPrefix = pRes.prefix;
+  }
+  return {
+    prefix: detectedPrefix,
+    stem: current,
+    suffix: detectedSuffix
+  };
+}
+function stemUrdu(word, options = {}) {
+  return getAffixes(word, options).stem;
+}
+function stemUrduText(text, options = {}) {
+  if (!text || typeof text !== "string") return "";
+  const tokens = text.split(/([\s\p{P}\p{S}]+)/u);
+  return tokens.map((token) => {
+    if (/[\u0600-\u06FF\u0750-\u077F\uFB50-\uFDFF\uFE70-\uFEFF]/.test(token)) {
+      return stemUrdu(token, options);
+    }
+    return token;
+  }).join("");
+}
+
 exports.URDU_MONTHS_GREGORIAN = URDU_MONTHS_GREGORIAN;
 exports.URDU_MONTHS_HIJRI = URDU_MONTHS_HIJRI;
+exports.URDU_PREFIXES = URDU_PREFIXES;
 exports.URDU_STOP_WORDS = URDU_STOP_WORDS;
+exports.URDU_SUFFIXES = URDU_SUFFIXES;
 exports.URDU_WEEKDAYS = URDU_WEEKDAYS;
 exports.analyzeUrdu = analyzeUrdu;
 exports.compareUrdu = compareUrdu;
@@ -3716,6 +3956,7 @@ exports.editDistance = editDistance;
 exports.filterStopWords = filterStopWords;
 exports.foldUrdu = foldUrdu;
 exports.formatUrduDate = formatUrduDate;
+exports.getAffixes = getAffixes;
 exports.getUrduMonthName = getUrduMonthName;
 exports.getUrduWeekdayName = getUrduWeekdayName;
 exports.hasUrduSpecificLetters = hasUrduSpecificLetters;
@@ -3734,6 +3975,8 @@ exports.searchUrduRanked = searchUrduRanked;
 exports.sortUrdu = sortUrdu;
 exports.splitSentences = splitSentences;
 exports.splitWords = splitWords;
+exports.stemUrdu = stemUrdu;
+exports.stemUrduText = stemUrduText;
 exports.timeAgoUrdu = timeAgoUrdu;
 exports.toArabicIndicDigits = toArabicIndicDigits;
 exports.toEnglishDigits = toEnglishDigits;