arbtok 0.0.0a2__py3-none-any.whl

arbtok/__init__.py

@@ -0,0 +1,13 @@
+"""
+I'm just a dude trying to create TTS voices to help the visually impaired arabic community.
+
+I am not an arabic speaker, I never had any previous interaction with the arabic script, this might be a pointless experiment powered by LLM hallucinations.
+
+You have been warned.
+
+if you speak MSA pull requests welcome!
+
+- test.py contains a reference dataset to benchmark the code against, IPA transcriptions have been LLM generated
+- main logic is in tokenizer.py
+- see tests folder for some benchmarks and comparison against pre existing (low quality) datasets
+"""

arbtok/constants.py

@@ -0,0 +1,87 @@
+import string
+from typing import Set
+
+# ==============================================================================
+# ARABIC CONSTANTS
+# ==============================================================================
+
+# --- Diacritics (Tashkeel) ---
+FATHA = '\u064E'  # َ (short a)
+DAMMA = '\u064F'  # ُ (short u)
+KASRA = '\u0650'  # ِ (short i)
+SHADDA = '\u0651'  # ّ (consonant doubling/gemination)
+SUKUN = '\u0652'  # ْ (no vowel/silence)
+TANWIN_FATH = '\u064B'  # ً (an)
+TANWIN_DAMM = '\u064C'  # ٌ (un)
+TANWIN_KASR = '\u064D'  # ٍ (in)
+DAGGER_ALIF = '\u0670'  # ٰ (superscript alef, long a)
+MADD = '\u0653'  # ٓ (madda, vowel elongation)
+
+# --- Letters & Variants ---
+HAMZA = '\u0621'  # ء (standalone hamza)
+ALEF_MADDA = '\u0622'  # آ (alef with madda)
+ALEF_HAMZA_ABOVE = '\u0623'  # أ (alef with hamza above)
+WAW_HAMZA = '\u0624'  # ؤ (waw with hamza)
+ALEF_HAMZA_BELOW = '\u0625'  # إ (alef with hamza below)
+YA_HAMZA = '\u0626'  # ئ (ya with hamza)
+ALIF = '\u0627'  # ا (bare alef)
+TA_MARBUTA = '\u0629'  # ة (tied ta, fem. marker)
+WAW = '\u0648'  # و
+ALIF_MAKSURA = '\u0649'  # ى (broken alef)
+YA = '\u064A'  # ي
+LAM = '\u0644'  # ل
+HAMZAT_AL_WASL = '\u0671'  # ٱ (wasl, connecting hamza)
+
+ALEF_VARIANTS = {ALIF, ALEF_MADDA, ALEF_HAMZA_ABOVE, ALEF_HAMZA_BELOW, HAMZAT_AL_WASL}
+
+# --- Punctuation ---
+ARABIC_PUNCT = "،؛؟ـ…"  # Comma, Semicolon, Question mark, Tatweel, Ellipsis
+PUNCT = string.punctuation + ARABIC_PUNCT
+
+# --- Sun Letters (Shamsiyya) ---
+# These consonants assimilate the 'l' of the definite article 'al-'.
+# e.g., al-shams -> ash-shams.
+SUN_LETTERS: Set[str] = {
+    '\u062A',  # ت (t)
+    '\u062B',  # ث (th)
+    '\u062F',  # د (d)
+    '\u0630',  # ذ (dh)
+    '\u0631',  # ر (r)
+    '\u0632',  # ز (z)
+    '\u0633',  # س (s)
+    '\u0634',  # ش (sh)
+    '\u0635',  # ص (sad)
+    '\u0636',  # ض (dad)
+    '\u0637',  # ط (ta)
+    '\u0638',  # ظ (dha)
+    '\u0644',  # ل (l)
+    '\u0646',  # ن (n)
+}
+
+# --- Proclitics ---
+# Short prepositions/conjunctions that attach to the following word in writing.
+CLITIC_BASES = {
+    WAW,  # و (and)
+    '\u0641',  # ف (so/then)
+    '\u0628',  # ب (by/with)
+    '\u0643',  # ك (like/as)
+    LAM,  # ل (for/to)
+    '\u0633'  # س (future marker 'sa-')
+}
+
+# --- IPA Mappings ---
+# for easy unambiguous phoneme reference in the code when the unicode maps to a specific IPA sound
+B = '\u0628'
+R = '\u0631'
+Z = '\u0632'
+S = '\u0633'
+F = '\u0641'
+Q = '\u0642'
+K = '\u0643'
+M = '\u0645'
+N = '\u0646'
+H = '\u0647'
+T = '\u062A'
+DJ = '\u062C'
+X = '\u062E'
+D = '\u062F'

arbtok/dialects.py

@@ -0,0 +1,111 @@
+"""
+TODO: to be extended with different phoneme realizations per dialect.
+logic in tokenizer might use ArabicDialect enum if needed for further contextual rules
+
+currently only MSA is targeted
+"""
+from enum import Enum
+from arbtok.constants import (B, T, DJ, X, D, R, Z, S, F, Q, K, M, N, H, LAM, WAW, YA,
+                              FATHA, DAMMA, KASRA, DAGGER_ALIF, MADD,
+                              HAMZA, ALEF_HAMZA_ABOVE, ALEF_HAMZA_BELOW, WAW_HAMZA, YA_HAMZA,
+                              TANWIN_FATH, TANWIN_KASR, TANWIN_DAMM,
+                              ALIF_MAKSURA)
+
+
+class ArabicDialect(str, Enum):
+    CLA = "CLA"
+    MSA = "MSA"
+
+
+ARABIC_TO_IPA_CONSONANTS = {
+    HAMZA: "ʔ", ALEF_HAMZA_ABOVE: "ʔ", ALEF_HAMZA_BELOW: "ʔ", WAW_HAMZA: "ʔ", YA_HAMZA: "ʔ",
+    B: 'b',
+    T: 't',
+    '\u062B': 'θ',
+    DJ: 'dʒ',  # DJ: 'ʤ',
+    '\u062D': 'ħ',
+    X: 'x',
+    D: 'd',
+    '\u0630': 'ð',
+    R: 'r',
+    Z: 'z',
+    S: 's',
+    '\u0634': 'ʃ',
+    '\u0635': 'sˤ',
+    '\u0636': 'dˤ',
+    '\u0637': 'tˤ',
+    '\u0638': 'ðˤ',
+    '\u0639': 'ʕ',
+    '\u063A': 'ɣ',
+    F: 'f',
+    Q: 'q',
+    K: 'k',
+    M: 'm',
+    N: 'n',
+    H: 'h',
+    LAM: 'l',
+    WAW: 'w',
+    YA: 'j',
+    # Other unicode variations
+    "ﻻ": "laː"
+}
+
+DIACRITIC_TO_IPA = {
+    FATHA: 'a',
+    DAMMA: 'u',
+    KASRA: 'i',
+    DAGGER_ALIF: 'aː',
+    MADD: 'aː'
+}
+TANWIN_TO_IPA = {
+    TANWIN_FATH: "an",
+    TANWIN_DAMM: "un",
+    TANWIN_KASR: "in"
+}
+
+VOWEL_MAP = {**DIACRITIC_TO_IPA,
+             **TANWIN_TO_IPA,
+             MADD: ":",
+             ALIF_MAKSURA: 'aː'}
+
+# --- Word Exceptions ---
+# TODO - LLM generated, needs validation from native speaker
+# Dictionary for words with irregular orthography vs pronunciation.
+# These words have "deficient" or "historical" spelling where vowels
+# are pronounced but not written, or written letters are silent.
+WORD_EXCEPTIONS = {
+    # === Unicode / orthographic variants ===
+
+    # Demonstratives with "Dagger Alif" (pronounced long /a:/ but written short or omitted)
+    "هَٰذَا": "haːðaː",  # ha-dha (this, m.)
+    "هٰذَا": "haːðaː",  # variant
+    "هَذَا": "haːðaː",  # common deficient spelling
+    "هَٰذِهِ": "haːðihi",  # ha-dhi-hi (this, f.)
+    "هٰذِهِ": "haːðihi",  # variant
+    "ذَٰلِكَ": "ðaːlika",  # dha-li-ka (that)
+    "ذٰلِكَ": "ðaːlika",  # variant
+    "أُولَٰئِكَ": "ʔulaːʔika",  # u-la-i-ka (those) - note medial hamza logic is complex, hardcoded here
+
+    # Particles
+    "لَٰكِن": "laːkin",  # la-kin (but) - unwritten medial alif
+    "لَكِن": "laːkin",  # deficient spelling
+    "لَٰكِنَّ": "laːkinna",  # la-kin-na (but...)
+
+    # Divine Names
+    "ﷲ": "allaːh",
+    "اللّٰه": "allaːh",  # Allah - heavy L, unwritten alif
+    "اللَّه": "allaːh",  # variant
+    "إِلَٰه": "ʔilaːh",  # ilah (god)
+    "الرَّحْمَٰن": "arraħmaːn",  # Ar-Rahman
+
+    # Irregular pronunciations
+    "مِائَة": "miʔa",  # mi'a (hundred) - silent extra Alif in spelling
+    "عَمْرٌو": "ʕamrun",  # 'Amr - final Waw is silent (differentiates from 'Umar)
+    "أُولِي": "ʔuliː",  # 'uli (owners of) - silent Waw
+    "أُولُو": "ʔuluː",  # 'ulu - silent Waw
+
+    # TODO - why are these irregular?
+    "مُسْتَشْفَى": "mustashfaː",
+    "مَدْرَسَة": "madrasah"
+}
+

arbtok/espeak_wrapper.py

@@ -0,0 +1,237 @@
+import os
+import re
+import string
+import subprocess
+from typing import List, Optional
+from typing import Tuple
+
+import numpy as np
+import onnxruntime
+import requests
+from quebra_frases import sentence_tokenize
+
+from arbtok.tashkeel import TashkeelDiacritizer
+from arbtok.util import normalize, match_lang
+
+# list of (substring, terminator, end_of_sentence) tuples.
+TextChunks = List[Tuple[str, str, bool]]
+# list of (phonemes, terminator, end_of_sentence) tuples.
+RawPhonemizedChunks = List[Tuple[str, str, bool]]
+
+PhonemizedChunks = list[list[str]]
+
+
+class EspeakError(Exception):
+    """Custom exception for espeak-ng related errors."""
+    pass
+
+
+class EspeakPhonemizer:
+    """
+    A phonemizer class that uses the espeak-ng command-line tool to convert text into phonemes.
+    It segments the input text heuristically based on punctuation to mimic clause-by-clause processing.
+    """
+    ESPEAK_LANGS = ['es-419', 'ca', 'qya', 'ga', 'et', 'ky', 'io', 'fa-latn', 'en-gb', 'fo', 'haw', 'kl',
+                    'ta', 'ml', 'gd', 'sd', 'es', 'hy', 'ur', 'ro', 'hi', 'or', 'ti', 'ca-va', 'om', 'tr', 'pa',
+                    'smj', 'mk', 'bg', 'cv', "fr", 'fi', 'en-gb-x-rp', 'ru', 'mt', 'an', 'mr', 'pap', 'vi', 'id',
+                    'fr-be', 'ltg', 'my', 'nl', 'shn', 'ba', 'az', 'cmn', 'da', 'as', 'sw',
+                    'piqd', 'en-us', 'hr', 'it', 'ug', 'th', 'mi', 'cy', 'ru-lv', 'ia', 'tt', 'hu', 'xex', 'te', 'ne',
+                    'eu', 'ja', 'bpy', 'hak', 'cs', 'en-gb-scotland', 'hyw', 'uk', 'pt', 'bn', 'mto', 'yue',
+                    'be', 'gu', 'sv', 'sl', 'cmn-latn-pinyin', 'lfn', 'lv', 'fa', 'sjn', 'nog', 'ms',
+                    'vi-vn-x-central', 'lt', 'kn', 'he', 'qu', 'ca-ba', 'quc', 'nb', 'sk', 'tn', 'py', 'si', 'de',
+                    'ar', 'en-gb-x-gbcwmd', 'bs', 'qdb', 'sq', 'sr', 'tk', 'en-029', 'ht', 'ru-cl', 'af', 'pt-br',
+                    'fr-ch', 'ka', 'en-gb-x-gbclan', 'ko', 'is', 'ca-nw', 'gn', 'kok', 'la', 'lb', 'am', 'kk', 'ku',
+                    'kaa', 'jbo', 'eo', 'uz', 'nci', 'vi-vn-x-south', 'el', 'pl', 'grc', ]
+
+    def __init__(self, pausal: bool = True):
+        self.pausal = pausal  # arabic only
+        self._tashkeel: Optional[TashkeelDiacritizer] = None
+
+    @classmethod
+    def get_lang(cls, target_lang: str) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        if target_lang.lower() == "en-gb":
+            return "en-gb-x-rp"
+        if target_lang in cls.ESPEAK_LANGS:
+            return target_lang
+        if target_lang.lower().split("-")[0] in cls.ESPEAK_LANGS:
+            return target_lang.lower().split("-")[0]
+        return cls.match_lang(target_lang, cls.ESPEAK_LANGS)
+
+    @property
+    def tashkeel(self) -> TashkeelDiacritizer:
+        if self._tashkeel is None:
+            self._tashkeel = TashkeelDiacritizer()
+        return self._tashkeel
+
+    @staticmethod
+    def _run_espeak_command(args: List[str], input_text: str = None, check: bool = True) -> str:
+        """
+        Helper function to run espeak-ng commands via subprocess.
+        Executes 'espeak-ng' with the given arguments and input text.
+        Captures stdout and stderr, and raises EspeakError on failure.
+
+        Args:
+            args (List[str]): A list of command-line arguments for espeak-ng.
+            input_text (str, optional): The text to pass to espeak-ng's stdin. Defaults to None.
+            check (bool, optional): If True, raises a CalledProcessError if the command returns a non-zero exit code. Defaults to True.
+
+        Returns:
+            str: The stripped standard output from the espeak-ng command.
+
+        Raises:
+            EspeakError: If espeak-ng command is not found, or if the subprocess call fails.
+        """
+        command: List[str] = ['espeak-ng'] + args
+
+        # Standard arguments for subprocess.run
+        subprocess_args = {
+            'input': input_text,
+            'capture_output': True,
+            'text': True,
+            'check': check,
+            'encoding': 'utf-8',
+            'errors': 'replace'  # Replaces unencodable characters with a placeholder
+        }
+
+        # Add 'creationflags' to hide the terminal window on Windows
+        if os.name == 'nt':
+            subprocess_args['creationflags'] = 0x08000000
+
+        try:
+            process: subprocess.CompletedProcess = subprocess.run(
+                command,
+                **subprocess_args  # Use the dynamic arguments
+            )
+            return process.stdout.strip()
+        except FileNotFoundError:
+            raise EspeakError(
+                "espeak-ng command not found. Please ensure espeak-ng is installed "
+                "and available in your system's PATH."
+            )
+        except subprocess.CalledProcessError as e:
+            raise EspeakError(
+                f"espeak-ng command failed with error code {e.returncode}:\n"
+                f"STDOUT: {e.stdout}\n"
+                f"STDERR: {e.stderr}"
+            )
+        except Exception as e:
+            raise EspeakError(f"An unexpected error occurred while running espeak-ng: {e}")
+
+    def phonemize_string(self, text: str, lang: str) -> str:
+        lang = self.get_lang(lang)
+        return self._run_espeak_command(
+            ['-q', '-x', '--ipa', '-v', lang],
+            input_text=text
+        )
+
+    def phonemize_to_list(self, text: str, lang: str = "ar") -> List[str]:
+        return list(self.phonemize_string(text, lang))
+
+    def add_diacritics(self, text: str, lang: str= "ar") -> str:
+        if lang.startswith("ar"):
+            return self.tashkeel.diacritize(text, pausal=self.pausal)
+        return text
+
+    def phonemize(self, text: str, lang: str= "ar") -> PhonemizedChunks:
+        if not text:
+            return [('', '', True)]
+        results: RawPhonemizedChunks = []
+        text = normalize(text, lang)
+        for chunk, punct, eos in self.chunk_text(text):
+            phoneme_str = self.phonemize_string(self.remove_punctuation(chunk), lang)
+            results += [(phoneme_str, punct, True)]
+        return self._process_phones(results)
+
+    @staticmethod
+    def _process_phones(raw_phones: RawPhonemizedChunks) -> PhonemizedChunks:
+        """Text to phonemes grouped by sentence."""
+        all_phonemes: list[list[str]] = []
+        sentence_phonemes: list[str] = []
+        for phonemes_str, terminator_str, end_of_sentence in raw_phones:
+            # Filter out (lang) switch (flags).
+            # These surround words from languages other than the current voice.
+            phonemes_str = re.sub(r"\([^)]+\)", "", phonemes_str)
+            sentence_phonemes.extend(list(phonemes_str))
+            if end_of_sentence:
+                all_phonemes.append(sentence_phonemes)
+                sentence_phonemes = []
+        if sentence_phonemes:
+            all_phonemes.append(sentence_phonemes)
+        return all_phonemes
+
+    @staticmethod
+    def match_lang(target_lang: str, valid_langs: List[str]) -> str:
+        """
+        Validates and returns the closest supported language code.
+
+        Args:
+            target_lang (str): The language code to validate.
+
+        Returns:
+            str: The validated language code.
+
+        Raises:
+            ValueError: If the language code is unsupported.
+        """
+        lang, score = match_lang(target_lang, valid_langs)
+        if score > 10:
+            # raise an error for unsupported language
+            raise ValueError(f"unsupported language code: {target_lang}")
+        return lang
+
+    @staticmethod
+    def remove_punctuation(text):
+        """
+        Removes all punctuation characters from a string.
+        Punctuation characters are defined by string.punctuation.
+        """
+        # Create a regex pattern that matches any character in string.punctuation
+        punctuation_pattern = r"[" + re.escape(string.punctuation) + r"]"
+        return re.sub(punctuation_pattern, '', text).strip()
+
+    @staticmethod
+    def chunk_text(text: str, delimiters: Optional[List[str]] = None) -> TextChunks:
+        if not text:
+            return [('', '', True)]
+
+        results: TextChunks = []
+        delimiters = delimiters or [":", ";", "...", "|"]
+
+        # Create a regex pattern that matches any of the delimiters
+        delimiter_pattern = re.escape(delimiters[0])
+        for delimiter in delimiters[1:]:
+            delimiter_pattern += f"|{re.escape(delimiter)}"
+
+        for sentence in sentence_tokenize(text):
+            # Default punctuation if no specific punctuation found
+            default_punc = sentence[-1] if sentence and sentence[-1] in string.punctuation else "."
+
+            # Use regex to split the sentence by any of the delimiters
+            parts = re.split(f'({delimiter_pattern})', sentence)
+
+            # Group parts into chunks (text + delimiter)
+            chunks = []
+            for i in range(0, len(parts), 2):
+                # If there's a delimiter after the text, use it
+                delimiter = parts[i + 1] if i + 1 < len(parts) else default_punc
+
+                # Last chunk is marked as complete
+                is_last = (i + 2 >= len(parts))
+
+                chunks.append((parts[i].strip(), delimiter.strip(), is_last))
+
+            results.extend(chunks)
+
+        return results

arbtok/num2words.py

@@ -0,0 +1,170 @@
+import re
+from functools import partial
+
+from arbtok.pyarabic import araby
+from arbtok.pyarabic import number as arnum
+from arbtok.pyarabic.trans import normalize_digits
+
+NUM_REGEX = re.compile(r"\d+")
+PERCENT_NO_DIAC = "بالمئة"
+PERCENT_DIAC = "بِالْمِئَة"
+
+
+def _convert_num2words(m: re.Match, *, apply_tashkeel):
+    number = m.group(0)
+    word_representation = arnum.number2text(number)
+    if apply_tashkeel:
+        return " ".join(arnum.pre_tashkeel_number(word_representation.split(" ")))
+    return word_representation
+
+
+def num2words(text: str, handle_percent=True, apply_tashkeel: bool = True) -> str:
+    """
+    Converts numbers in `text` to Arabic words.
+    Simple conversion. Does not check if the number is date/currency...etc.
+
+    Args:
+        text: input text that may contain numbers
+        apply_tashkeel: diacritize added words
+    """
+    text = normalize_digits(text)
+    output = NUM_REGEX.sub(
+        partial(_convert_num2words, apply_tashkeel=apply_tashkeel), text
+    )
+    if handle_percent:
+        replacement = PERCENT_DIAC if apply_tashkeel else PERCENT_NO_DIAC
+        output = output.replace("%", f" {replacement}")
+    return araby.fix_spaces(output)
+
+# ------------------------------------------------------
+# FEMININE HEAD INFLECTION
+# ------------------------------------------------------
+
+FEM_HEAD = {
+    "واحد": "واحدة",
+    "اثنان": "اثنتان",
+    "اثنين": "اثنتين",
+    "اثنانِ": "اثنتين",
+    "ثلاثة": "ثلاث",
+    "أربعة": "أربع",
+    "خمسة": "خمس",
+    "ستة": "ست",
+    "سبعة": "سبع",
+    "ثمانية": "ثمان",
+    "تسعة": "تسع",
+}
+
+TEENS_FEM = {
+    "أحد عشر": "إحدى عشرة",
+    "اثنا عشر": "اثنتا عشرة",
+    "اثني عشر": "اثنتي عشرة",
+    "ثلاثة عشر": "ثلاث عشرة",
+    "أربعة عشر": "أربع عشرة",
+    "خمسة عشر": "خمس عشرة",
+    "ستة عشر": "ست عشرة",
+    "سبعة عشر": "سبع عشرة",
+    "ثمانية عشر": "ثماني عشرة",
+    "تسعة عشر": "تسع عشرة",
+}
+
+CASE_M = {
+    "nom": "ٌ",
+    "acc": "ً",
+    "gen": "ٍ",
+}
+
+# ------------------------------------------------------
+# ORDINALS
+# ------------------------------------------------------
+
+ORDINAL_BASE = {
+    1: ("الأوّل", "الأولى"),
+    2: ("الثاني", "الثانية"),
+    3: ("الثالث", "الثالثة"),
+    4: ("الرابع", "الرابعة"),
+    5: ("الخامس", "الخامسة"),
+    6: ("السادس", "السادسة"),
+    7: ("السابع", "السابعة"),
+    8: ("الثامن", "الثامنة"),
+    9: ("التاسع", "التاسعة"),
+    10: ("العاشر", "العاشرة"),
+    11: ("الحادي عشر", "الحادية عشرة"),
+    12: ("الثاني عشر", "الثانية عشرة"),
+    13: ("الثالث عشر", "الثالثة عشرة"),
+    14: ("الرابع عشر", "الرابعة عشرة"),
+    15: ("الخامس عشر", "الخامسة عشرة"),
+    16: ("السادس عشر", "السادسة عشرة"),
+    17: ("السابع عشر", "السابعة عشرة"),
+    18: ("الثامن عشر", "الثامنة عشرة"),
+    19: ("التاسع عشر", "التاسعة عشرة"),
+    20: ("العشرون", "العشرون"),
+}
+
+def ordinal_of_number(n: int, feminine=False):
+    if n in ORDINAL_BASE:
+        return ORDINAL_BASE[n][1 if feminine else 0]
+    # Synthetic fallback for 21–99
+    # Prepend definite article to the cardinals and add ي/ية to the head token.
+    base = arnum.number2text(n).split()
+    head = araby.strip_tashkeel(base[-1])
+    if feminine:
+        base[-1] = head + "ية"
+    else:
+        base[-1] = head + "ي"
+    return "ال" + " ".join(base)
+
+# ------------------------------------------------------
+# HEAD EXTRACTION + INFLECTION
+# ------------------------------------------------------
+
+def extract_head(tokens):
+    return tokens[-1], len(tokens) - 1
+
+def apply_head_feminine(tokens):
+    phrase = " ".join(tokens)
+    if phrase in TEENS_FEM:
+        return TEENS_FEM[phrase]
+
+    head, idx = extract_head(tokens)
+    pure = araby.strip_tashkeel(head)
+    if pure in FEM_HEAD:
+        tokens[idx] = FEM_HEAD[pure]
+    return " ".join(tokens)
+
+def apply_head_case(tokens, case):
+    head, idx = extract_head(tokens)
+    pure = araby.strip_tashkeel(head)
+    tokens[idx] = pure + CASE_M[case]
+    return " ".join(tokens)
+
+# ------------------------------------------------------
+# FULL GENERATION
+# ------------------------------------------------------
+
+def generate_variants_for_number(i: int, apply_tashkeel=False):
+    base = num2words(str(i), handle_percent=False, apply_tashkeel=apply_tashkeel)
+    tokens = base.split()
+
+    # Cardinals
+    masc = base
+    nom = apply_head_case(tokens.copy(), "nom")
+    acc = apply_head_case(tokens.copy(), "acc")
+    gen = apply_head_case(tokens.copy(), "gen")
+    fem = apply_head_feminine(tokens.copy())
+
+    # Ordinals (masc + fem)
+    try:
+        n = int(i)
+        ordm = ordinal_of_number(n, feminine=False)
+        ordf = ordinal_of_number(n, feminine=True)
+    except:
+        ordm = None
+        ordf = None
+
+    out = []
+
+    for w in [masc, nom, acc, gen, fem, ordm, ordf]:
+        if w:
+            out.append(w)
+
+    return sorted(set(out))