morphbpe-pl 0.3.0__py3-none-any.whl

morphbpe/__init__.py

@@ -0,0 +1,22 @@
+"""MorphBPE: Morphologically-aware BPE tokenizer for Polish."""
+
+__version__ = "0.3.0"
+
+from morphbpe.tokenizer import (
+    MorphBPETokenizer,
+    pre_tokenize,
+    SPECIAL_TOKENS,
+    BYTE_TOKENS,
+    BYTE_TOKENS_INV,
+)
+from morphbpe.morph_metrics import evaluate_all, evaluate_lexeme
+
+__all__ = [
+    "MorphBPETokenizer",
+    "pre_tokenize",
+    "SPECIAL_TOKENS",
+    "BYTE_TOKENS",
+    "BYTE_TOKENS_INV",
+    "evaluate_all",
+    "evaluate_lexeme",
+]

morphbpe/cli.py

@@ -0,0 +1,84 @@
+"""Command-line interface for MorphBPE."""
+
+import argparse
+import sys
+
+from morphbpe.tokenizer import MorphBPETokenizer
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        prog="morphbpe",
+        description="MorphBPE: Morphologically-aware BPE tokenizer for Polish",
+    )
+    sub = parser.add_subparsers(dest="command", required=True)
+
+    # ── train ────────────────────────────────────────────────────────
+    p_train = sub.add_parser("train", help="Train a tokenizer from a text file")
+    p_train.add_argument("corpus", help="Path to plain-text corpus (one doc per line)")
+    p_train.add_argument("-v", "--vocab-size", type=int, default=1000)
+    p_train.add_argument("-a", "--alpha", type=float, default=2.0)
+    p_train.add_argument(
+        "-c", "--constraint", choices=["suffix", "prefix", "both"], default="suffix"
+    )
+    p_train.add_argument(
+        "-o", "--output", default="morphbpe-model", help="Output directory"
+    )
+
+    # ── encode ───────────────────────────────────────────────────────
+    p_enc = sub.add_parser("encode", help="Encode text using a trained tokenizer")
+    p_enc.add_argument("model", help="Tokenizer directory")
+    p_enc.add_argument(
+        "text", nargs="?", default=None, help="Text to encode (or stdin)"
+    )
+
+    # ── info ─────────────────────────────────────────────────────────
+    p_info = sub.add_parser("info", help="Show tokenizer info")
+    p_info.add_argument("model", help="Tokenizer directory")
+
+    args = parser.parse_args()
+
+    if args.command == "train":
+        _cmd_train(args)
+    elif args.command == "encode":
+        _cmd_encode(args)
+    elif args.command == "info":
+        _cmd_info(args)
+
+
+def _cmd_train(args):
+    tok = MorphBPETokenizer(
+        vocab_size=args.vocab_size,
+        alpha=args.alpha,
+        constraint=args.constraint,
+    )
+    tok.train_from_file(args.corpus)
+    tok.save(args.output)
+    print(f"Saved to {args.output}/")
+
+
+def _cmd_encode(args):
+    tok = MorphBPETokenizer.load(args.model)
+    if args.text:
+        lines = [args.text]
+    else:
+        lines = sys.stdin.read().splitlines()
+
+    for line in lines:
+        tokens = []
+        for word in line.split():
+            tokens.extend(tok.encode_word(word))
+        print(" ".join(tokens))
+
+
+def _cmd_info(args):
+    tok = MorphBPETokenizer.load(args.model)
+    print(f"Vocab size:  {tok.get_vocab_size()}")
+    print(f"Merges:      {len(tok.merges)}")
+    print(f"Alpha:       {tok.alpha}")
+    print(f"Constraint:  {tok.constraint}")
+    print(f"Morfeusz:    {tok._backend.has_morfeusz}")
+
+
+if __name__ == "__main__":
+    main()

morphbpe/hf_tokenizer.py

@@ -0,0 +1,232 @@
+"""HuggingFace tokenizer wrapper for MorphBPE.
+
+Provides a PreTrainedTokenizer-compatible interface so MorphBPE
+can be used with HuggingFace transformers pipelines.
+
+Usage:
+    from morphbpe.hf_tokenizer import MorphBPEHFTokenizer
+
+    tok = MorphBPEHFTokenizer.from_pretrained("path/to/model")
+    ids = tok("prezydent podpisał ustawę")
+"""
+
+import json
+from pathlib import Path
+from typing import Dict, List, Optional, Union
+
+from morphbpe.tokenizer import MorphBPETokenizer, SPECIAL_TOKENS
+
+
+class MorphBPEHFTokenizer:
+    """Lightweight HF-compatible wrapper around MorphBPETokenizer.
+
+    Implements the core interface needed by transformers:
+    - __call__(text) -> dict with input_ids, attention_mask
+    - encode(text) -> List[int]
+    - decode(ids) -> str
+    - batch_encode(texts) -> dict with padded batches
+    - save_pretrained(dir) / from_pretrained(dir)
+    - get_vocab() / get_vocab_size()
+    - __len__() / __contains__()
+    """
+
+    def __init__(self, tokenizer: MorphBPETokenizer):
+        self._tok = tokenizer
+
+    @classmethod
+    def from_pretrained(cls, path: str) -> "MorphBPEHFTokenizer":
+        tok = MorphBPETokenizer.load(path)
+        return cls(tok)
+
+    def save_pretrained(self, directory: str) -> None:
+        self._tok.save(directory)
+        config_path = Path(directory) / "tokenizer_config.json"
+        config = {
+            "tokenizer_class": "MorphBPEHFTokenizer",
+            "model_type": "morphbpe",
+            "vocab_size": len(self._tok),
+            "alpha": self._tok.alpha,
+            "constraint": self._tok.constraint,
+            "pad_token": "<pad>",
+            "unk_token": "<unk>",
+            "cls_token": "<cls>",
+            "sep_token": "<sep>",
+            "mask_token": "<mask>",
+            "pad_token_id": self._tok.pad_token_id,
+            "unk_token_id": self._tok.unk_token_id,
+            "cls_token_id": self._tok.cls_token_id,
+            "sep_token_id": self._tok.sep_token_id,
+            "mask_token_id": self._tok.mask_token_id,
+        }
+        config_path.write_text(
+            json.dumps(config, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+
+    def __call__(
+        self,
+        text: Union[str, List[str]],
+        add_special_tokens: bool = True,
+        padding: Union[bool, str] = True,
+        truncation: bool = False,
+        max_length: Optional[int] = None,
+        return_tensors: Optional[str] = None,
+        return_attention_mask: bool = True,
+        return_token_type_ids: bool = False,
+        **kwargs,
+    ) -> dict:
+        """Encode text(s) and return dict compatible with transformers.
+
+        Parameters
+        ----------
+        text : str or List[str]
+            Input text(s).
+        add_special_tokens : bool
+            Prepend [CLS] and append [SEP].
+        padding : bool or str
+            If True/"longest", pad to longest in batch. If "max_length", pad to max_length.
+        truncation : bool
+            If True, truncate to max_length.
+        max_length : int, optional
+            Maximum sequence length.
+        return_tensors : str, optional
+            If "pt", return PyTorch tensors.
+        return_attention_mask : bool
+            If True, include attention_mask.
+        return_token_type_ids : bool
+            If True, include token_type_ids (all zeros for single sentence).
+
+        Returns
+        -------
+        dict
+            {"input_ids": ..., "attention_mask": ..., "token_type_ids": ...}
+        """
+        is_batch = isinstance(text, list)
+
+        if is_batch:
+            result = self._tok.batch_encode(
+                text,
+                add_special_tokens=add_special_tokens,
+                padding=bool(padding),
+                max_length=max_length,
+                return_attention_mask=return_attention_mask,
+            )
+        else:
+            ids = self._tok.encode(text, add_special_tokens=add_special_tokens)
+            if truncation and max_length:
+                ids = ids[:max_length]
+            result = {"input_ids": ids}
+            if return_attention_mask:
+                result["attention_mask"] = [1] * len(ids)
+            if padding and not is_batch:
+                # Single text: no padding needed unless max_length specified
+                if max_length and len(ids) < max_length:
+                    pad_len = max_length - len(ids)
+                    result["input_ids"] = ids + [self._tok.pad_token_id] * pad_len
+                    if return_attention_mask:
+                        result["attention_mask"] = [1] * len(ids) + [0] * pad_len
+
+        if return_token_type_ids:
+            if is_batch:
+                result["token_type_ids"] = [
+                    [0] * len(ids) for ids in result["input_ids"]
+                ]
+            else:
+                result["token_type_ids"] = [0] * len(result["input_ids"])
+
+        # Convert to tensors if requested
+        if return_tensors == "pt":
+            try:
+                import torch
+
+                for key in result:
+                    if isinstance(result[key], list):
+                        if isinstance(result[key][0], list):
+                            result[key] = torch.tensor(result[key])
+                        else:
+                            result[key] = torch.tensor([result[key]])
+            except ImportError:
+                pass
+        elif return_tensors == "np":
+            try:
+                import numpy as np
+
+                for key in result:
+                    if isinstance(result[key], list):
+                        result[key] = np.array(result[key])
+            except ImportError:
+                pass
+
+        return result
+
+    def encode(
+        self,
+        text: str,
+        add_special_tokens: bool = True,
+    ) -> List[int]:
+        return self._tok.encode(text, add_special_tokens=add_special_tokens)
+
+    def decode(
+        self,
+        ids: List[int],
+        skip_special_tokens: bool = True,
+    ) -> str:
+        return self._tok.decode(ids, skip_special_tokens=skip_special_tokens)
+
+    def batch_encode(
+        self,
+        texts: List[str],
+        add_special_tokens: bool = True,
+        padding: bool = True,
+        max_length: Optional[int] = None,
+    ) -> Dict[str, List[List[int]]]:
+        return self._tok.batch_encode(
+            texts,
+            add_special_tokens=add_special_tokens,
+            padding=padding,
+            max_length=max_length,
+        )
+
+    def get_vocab(self) -> Dict[str, int]:
+        return self._tok.vocab
+
+    def get_vocab_size(self) -> int:
+        return self._tok.get_vocab_size()
+
+    def __len__(self) -> int:
+        return len(self._tok)
+
+    def __contains__(self, token: str) -> bool:
+        return token in self._tok
+
+    @property
+    def model_max_length(self) -> int:
+        return 2048
+
+    @property
+    def pad_token_id(self) -> int:
+        return self._tok.pad_token_id
+
+    @property
+    def unk_token_id(self) -> int:
+        return self._tok.unk_token_id
+
+    @property
+    def cls_token_id(self) -> int:
+        return self._tok.cls_token_id
+
+    @property
+    def sep_token_id(self) -> int:
+        return self._tok.sep_token_id
+
+    @property
+    def mask_token_id(self) -> int:
+        return self._tok.mask_token_id
+
+    def convert_ids_to_tokens(self, ids: List[int]) -> List[str]:
+        return [self._tok.id2token.get(i, "<unk>") for i in ids]
+
+    def convert_tokens_to_ids(self, tokens: List[str]) -> List[int]:
+        return [self._tok.token2id.get(t, self._tok.unk_token_id) for t in tokens]
+
+    def convert_tokens_to_string(self, tokens: List[str]) -> str:
+        return "".join(tokens)

morphbpe/morfeusz_backend.py

@@ -0,0 +1,86 @@
+"""Morpheme boundary detection: Morfeusz2 with heuristic fallback."""
+
+from typing import Dict, Set, Tuple
+
+try:
+    import morfeusz2
+    MORFEUSZ_AVAILABLE = True
+except ImportError:
+    MORFEUSZ_AVAILABLE = False
+
+# Heuristic suffix/prefix lists for when Morfeusz2 is unavailable
+_HEURISTIC_PREFIXES = [
+    "nie", "prze", "roz", "pod", "nad", "bez", "naj", "przy",
+]
+_HEURISTIC_SUFFIXES = [
+    "owanie", "ować", "owy", "owa", "owe",
+    "ych", "ego", "emu", "ami", "ach",
+    "ów", "em", "ę", "ą", "ć",
+]
+
+
+class MorfeuszBackend:
+    """Detects morpheme boundaries using Morfeusz2 or heuristic fallback."""
+
+    def __init__(self, use_morfeusz: bool = True):
+        self._morfeusz = None
+        if use_morfeusz and MORFEUSZ_AVAILABLE:
+            self._morfeusz = morfeusz2.Morfeusz()
+
+    @property
+    def has_morfeusz(self) -> bool:
+        return self._morfeusz is not None
+
+    def get_boundaries(self, word: str) -> Tuple[Set[int], Set[int]]:
+        """Return (prefix_boundaries, suffix_boundaries) as character positions."""
+        if self._morfeusz is not None:
+            return self._morfeusz_boundaries(word)
+        return self._heuristic_boundaries(word)
+
+    def get_boundaries_batch(
+        self, words: list[str]
+    ) -> Dict[str, Tuple[Set[int], Set[int]]]:
+        """Compute boundaries for a list of unique words."""
+        return {w: self.get_boundaries(w) for w in words}
+
+    # ── Morfeusz2 path ──────────────────────────────────────────────
+
+    def _morfeusz_boundaries(self, word: str) -> Tuple[Set[int], Set[int]]:
+        prefix_b: Set[int] = set()
+        suffix_b: Set[int] = set()
+        analysis = self._morfeusz.analyse(word)
+        if not analysis:
+            return prefix_b, suffix_b
+
+        interp = analysis[0]
+        lemma = interp[2][1] if len(interp[2]) > 1 else interp[2][0]
+        if not lemma or len(lemma) >= len(word):
+            return prefix_b, suffix_b
+
+        if word.startswith(lemma):
+            suffix_b.add(len(lemma))
+        else:
+            common = 0
+            for i in range(min(len(lemma), len(word))):
+                if lemma[i] == word[i]:
+                    common = i + 1
+                else:
+                    break
+            if 2 < common < len(word):
+                prefix_b.add(common)
+
+        return prefix_b, suffix_b
+
+    # ── Heuristic fallback ──────────────────────────────────────────
+
+    @staticmethod
+    def _heuristic_boundaries(word: str) -> Tuple[Set[int], Set[int]]:
+        prefix_b: Set[int] = set()
+        suffix_b: Set[int] = set()
+        for pfx in _HEURISTIC_PREFIXES:
+            if word.startswith(pfx) and len(word) > len(pfx) + 2:
+                prefix_b.add(len(pfx))
+        for sfx in _HEURISTIC_SUFFIXES:
+            if word.endswith(sfx) and len(word) > len(sfx) + 2:
+                suffix_b.add(len(word) - len(sfx))
+        return prefix_b, suffix_b

morphbpe/morph_metrics.py

@@ -0,0 +1,225 @@
+"""Morphological metrics: SI, ISS, MFL for tokenizer evaluation.
+
+Metrics:
+- SI  (Stem Integrity): fraction of forms where the stem is preserved as a single token
+- ISS (Inflectional Suffix Separation): fraction of forms where the inflectional
+        suffix is a separate token (not merged with stem)
+- MFL (Mean Fragmentation per Lexeme): average number of tokens per form,
+        averaged across all forms of a lexeme
+"""
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional, Tuple
+
+from morphbpe.morfeusz_backend import MorfeuszBackend
+
+
+@dataclass
+class LexemeResult:
+    """Per-lexeme evaluation result."""
+    lemma: str
+    forms: List[str]
+    mfl: float = 0.0
+    si: float = 0.0
+    iss: float = 0.0
+    per_form: List[dict] = field(default_factory=list)
+
+
+# ── Polish test paradigms ──────────────────────────────────────────
+
+# 10 lexemes: 5 verbs (incl. suppletive iść, być) + 5 nouns
+MORPH_PARADIGMS: Dict[str, List[str]] = {
+    # Verbs
+    "robić": [
+        "robię", "robisz", "robi", "robimy", "robicie", "robią",
+        "robiłem", "robiłeś", "robił", "robiłam", "robiłaś", "robiła",
+        "robiłem", "robiliśmy", "robiliście", "robili",
+        "rób", "róbmy", "róbcie",
+        "robić",
+    ],
+    "pisać": [
+        "piszę", "piszesz", "pisze", "piszemy", "piszecie", "piszą",
+        "pisałem", "pisałeś", "pisał", "pisałam", "pisałaś", "pisała",
+        "pisaliśmy", "pisaliście", "pisali",
+        "pisz", "piszmy", "piszcie",
+        "pisać",
+    ],
+    "czytać": [
+        "czytam", "czytasz", "czyta", "czytamy", "czytacie", "czytają",
+        "czytałem", "czytałeś", "czytał", "czytałam", "czytałaś", "czytała",
+        "czytaliśmy", "czytaliście", "czytali",
+        "czytaj", "czytajmy", "czytajcie",
+        "czytać",
+    ],
+    # Suppletive
+    "iść": [
+        "idę", "idziesz", "idzie", "idziemy", "idziecie", "idą",
+        "szedłem", "szedłeś", "szedł", "szłam", "szłaś", "szła",
+        "szliśmy", "szliście", "szli",
+        "idź", "idźmy", "idźcie",
+        "iść",
+    ],
+    "być": [
+        "jestem", "jesteś", "jest", "jesteśmy", "jesteście", "są",
+        "byłem", "byłeś", "był", "byłam", "byłaś", "była",
+        "byliśmy", "byliście", "byli",
+        "bądź", "bądźmy", "bądźcie",
+        "być",
+    ],
+    # Nouns
+    "ustawa": [
+        "ustawa", "ustawy", "ustawie", "ustawę", "ustawą", "ustawo",
+        "ustawy", "ustaw", "ustawom", "ustawy", "ustawami", "ustawach",
+    ],
+    "człowiek": [
+        "człowiek", "człowieka", "człowiekowi", "człowieka", "człowiekiem", "człowieku",
+        "ludzie", "ludzi", "ludziom", "ludzi", "ludźmi", "ludziach",
+    ],
+    "dziecko": [
+        "dziecko", "dziecka", "dziecku", "dziecko", "dzieckiem", "dziecku",
+        "dzieci", "dzieci", "dzieciom", "dzieci", "dziećmi", "dzieciach",
+    ],
+    "prawo": [
+        "prawo", "prawa", "prawu", "prawo", "prawem", "prawie",
+        "prawa", "praw", "prawom", "prawa", "prawami", "prawach",
+    ],
+    "kraj": [
+        "kraj", "kraju", "krajowi", "kraj", "krajem", "kraju",
+        "kraje", "krajów", "krajom", "kraje", "krajami", "krajach",
+    ],
+}
+
+
+def _find_stem(lemma: str, form: str) -> Tuple[str, str, str]:
+    """Heuristic stem extraction: longest common prefix of lemma and form.
+
+    Returns (prefix, stem, suffix) of the form.
+    prefix = characters before stem (empty for most Polish forms)
+    stem   = shared initial substring
+    suffix = remaining characters after stem
+    """
+    common = 0
+    for i in range(min(len(lemma), len(form))):
+        if lemma[i] == form[i]:
+            common = i + 1
+        else:
+            break
+    if common < 2:
+        # Suppletive — no shared stem
+        return "", form, ""
+    stem = form[:common]
+    suffix = form[common:]
+    return "", stem, suffix
+
+
+def evaluate_lexeme(lemma: str, forms: List[str], encode_word_fn) -> LexemeResult:
+    """Evaluate a single lexeme using morphological metrics.
+
+    Parameters
+    ----------
+    lemma : str
+        Dictionary form of the lexeme.
+    forms : list of str
+        All inflected forms to evaluate.
+    encode_word_fn : callable
+        Function that takes a word and returns list of subword tokens.
+    """
+    unique_forms = list(dict.fromkeys(forms))  # deduplicate, preserve order
+    total_tokens = 0
+    stem_intact_count = 0
+    suffix_sep_count = 0
+    per_form = []
+
+    for form in unique_forms:
+        tokens = encode_word_fn(form)
+        n_tokens = len(tokens)
+        total_tokens += n_tokens
+
+        _, stem, suffix = _find_stem(lemma, form)
+
+        # SI: is the stem a single token (or contained in one token)?
+        stem_intact = any(stem in t for t in tokens) if stem else False
+
+        # ISS: is the suffix a separate token?
+        if suffix:
+            suffix_sep = suffix in tokens
+        else:
+            suffix_sep = True  # no suffix to separate
+
+        if stem_intact:
+            stem_intact_count += 1
+        if suffix_sep:
+            suffix_sep_count += 1
+
+        per_form.append({
+            "form": form,
+            "tokens": tokens,
+            "n_tokens": n_tokens,
+            "stem": stem,
+            "suffix": suffix,
+            "stem_intact": stem_intact,
+            "suffix_sep": suffix_sep,
+        })
+
+    n = len(unique_forms)
+    return LexemeResult(
+        lemma=lemma,
+        forms=unique_forms,
+        mfl=total_tokens / n if n else 0.0,
+        si=stem_intact_count / n if n else 0.0,
+        iss=suffix_sep_count / n if n else 0.0,
+        per_form=per_form,
+    )
+
+
+def evaluate_all(
+    encode_word_fn,
+    paradigms: Optional[Dict[str, List[str]]] = None,
+) -> Tuple[List[LexemeResult], Dict[str, float]]:
+    """Evaluate tokenizer on all morphological paradigms.
+
+    Returns
+    -------
+    results : list of LexemeResult
+    summary : dict with macro-averaged SI, ISS, MFL
+    """
+    if paradigms is None:
+        paradigms = MORPH_PARADIGMS
+
+    results = []
+    for lemma, forms in paradigms.items():
+        r = evaluate_lexeme(lemma, forms, encode_word_fn)
+        results.append(r)
+
+    n = len(results)
+    summary = {
+        "SI": sum(r.si for r in results) / n if n else 0.0,
+        "ISS": sum(r.iss for r in results) / n if n else 0.0,
+        "MFL": sum(r.mfl for r in results) / n if n else 0.0,
+        "n_lexemes": n,
+        "n_forms": sum(len(r.forms) for r in results),
+    }
+    return results, summary
+
+
+def print_report(results: List[LexemeResult], summary: Dict[str, float]) -> str:
+    """Format a readable report string."""
+    lines = []
+    lines.append("Morphological Metrics Report")
+    lines.append("=" * 70)
+    lines.append(
+        f"{'Lexeme':<15} {'Forms':>5} {'MFL':>6} {'SI':>6} {'ISS':>6}"
+    )
+    lines.append("-" * 70)
+    for r in results:
+        lines.append(
+            f"{r.lemma:<15} {len(r.forms):>5} "
+            f"{r.mfl:>6.2f} {r.si:>6.1%} {r.iss:>6.1%}"
+        )
+    lines.append("-" * 70)
+    lines.append(
+        f"{'MACRO AVG':<15} {summary['n_forms']:>5} "
+        f"{summary['MFL']:>6.2f} {summary['SI']:>6.1%} {summary['ISS']:>6.1%}"
+    )
+    lines.append("=" * 70)
+    return "\n".join(lines)