mawo-razdel 1.0.3__py3-none-any.whl → 1.0.5__py3-none-any.whl

mawo_razdel/__init__.py

@@ -1,28 +1,26 @@
 """MAWO RAZDEL - Enhanced Russian Tokenization
-Upgraded tokenization with SynTagRus patterns for better sentence segmentation.
+Upgraded tokenization with 100% compatibility with original razdel.
 
 Features:
-- SynTagRus-based patterns (+25% quality on news)
+- Full backward compatibility with razdel API
+- All original razdel features preserved
+- Additional SynTagRus patterns available
 - Abbreviation handling (г., ул., им., т.д.)
 - Initials support (А. С. Пушкин)
 - Direct speech patterns
-- Backward compatible API
 """
 
 from __future__ import annotations
 
-import re
-from typing import Any
+# Import original razdel implementation (ported)
+from .segmenters import sentenize as _original_sentenize
+from .segmenters import tokenize as _original_tokenize
 
-# Try to import enhanced patterns
-try:
-    from .syntagrus_patterns import get_syntagrus_patterns
-
-    ENHANCED_PATTERNS_AVAILABLE = True
-except ImportError:
-    ENHANCED_PATTERNS_AVAILABLE = False
+# Import classes from substring module
+from .substring import Substring
 
 
+# Backwards compatibility aliases
 class Token:
     """Token with position information."""
 
@@ -51,219 +49,38 @@ class Sentence:
         )
 
 
-# Backwards compatibility alias
-class Substring:
-    """Backwards compatibility class for old tests."""
-
-    def __init__(self, start: int, stop: int, text: str) -> None:
-        self.start = start
-        self.stop = stop
-        self.text = text
-
-    def __repr__(self) -> str:
-        return (
-            f"Substring('{self.text[:30]}...')"
-            if len(self.text) > 30
-            else f"Substring('{self.text}')"
-        )
-
-
-def tokenize(text: str, use_enhanced: bool = True) -> list[Substring]:
-    """Токенизация русского текста.
-
-    Улучшенная токенизация с правильной обработкой:
-    - Десятичных чисел (3.14, 3,14)
-    - Процентов (95.5%)
-    - Диапазонов (1995-1999, 10:30-11:00)
-    - Дробей (1/2, 3/4)
-    - Телефонов, ID и т.д.
-
-    Args:
-        text: Текст для токенизации
-        use_enhanced: Использовать улучшенные паттерны
-
-    Returns:
-        Список объектов Substring (токенов)
-    """
-    # Улучшенный паттерн на основе современных практик NLP (2024-2025)
-    # Сохраняет целостность чисел при обработке русского текста
-    pattern = r"""
-        # Десятичные числа с точкой или запятой (3.14159 или 3,14159)
-        \d+[.,]\d+
-        # Диапазоны и временные интервалы (1995-1999, 10:30-11:00)
-        |\d+[-:]\d+(?:[-:]\d+)*
-        # Дроби (1/2, 3/4)
-        |\d+/\d+
-        # Проценты (с числом)
-        |\d+\s*%
-        # Обычные числа
-        |\d+
-        # Русские и латинские слова (включая ё)
-        |[\w\u0400-\u04FF]+
-        # Любой другой непробельный символ
-        |\S
-    """
-
-    tokens: list[Substring] = []
-    for match in re.finditer(pattern, text, re.VERBOSE | re.UNICODE):
-        token_text = match.group()
-        # Пропускаем чистые пробелы (не должно совпадать, но проверяем)
-        if token_text.strip():
-            tokens.append(Substring(match.start(), match.end(), token_text))
-
-    return tokens
-
-
-def sentenize(text: str, use_enhanced: bool = True) -> list[Sentence]:
-    """Segment Russian text into sentences.
-
-    Args:
-        text: Text to segment
-        use_enhanced: Use SynTagRus enhanced patterns (recommended)
-
-    Returns:
-        List of Sentence objects
-    """
-    if use_enhanced and ENHANCED_PATTERNS_AVAILABLE:
-        return _enhanced_sentenize(text)
-
-    # Fallback: simple segmentation
-    return _simple_sentenize(text)
+# Main API functions - use original razdel implementation
+def tokenize(text: str):
+    """Tokenize Russian text using original razdel algorithm.
 
+    Returns an iterator of Substring objects.
 
-def _enhanced_sentenize(text: str) -> list[Substring]:
-    """Enhanced sentence segmentation with SynTagRus patterns.
+    Examples:
+        >>> list(tokenize('что-то'))
+        [Substring(0, 6, 'что-то')]
 
-    Handles:
-    - Abbreviations (г., ул., т.д.)
-    - Initials (А. С. Пушкин)
-    - Direct speech
-    - Decimal numbers
+        >>> list(tokenize('1,5'))
+        [Substring(0, 3, '1,5')]
     """
-    patterns = get_syntagrus_patterns()
-
-    # Find sentence boundaries
-    boundaries = patterns.find_sentence_boundaries(text)
-
-    if not boundaries:
-        # No boundaries found, return whole text
-        clean_text = text.strip()
-        return [Substring(0, len(clean_text), clean_text)]
-
-    # Split by boundaries
-    sentences = []
-    start = 0
-
-    for boundary in boundaries:
-        sentence_text = text[start:boundary].strip()
-        if sentence_text:
-            # Find actual start position (skip leading whitespace)
-            actual_start = start + len(text[start:boundary]) - len(text[start:boundary].lstrip())
-            sentences.append(
-                Substring(actual_start, actual_start + len(sentence_text), sentence_text)
-            )
-        start = boundary
+    return _original_tokenize(text)
 
-    # Last sentence
-    if start < len(text):
-        sentence_text = text[start:].strip()
-        if sentence_text:
-            actual_start = start + len(text[start:]) - len(text[start:].lstrip())
-            sentences.append(
-                Substring(actual_start, actual_start + len(sentence_text), sentence_text)
-            )
 
-    return sentences
+def sentenize(text: str):
+    """Segment Russian text into sentences using original razdel algorithm.
 
+    Returns an iterator of Substring objects.
 
-def _simple_sentenize(text: str) -> list[Substring]:
-    """Simple sentence segmentation (fallback).
+    Examples:
+        >>> list(sentenize('Привет. Как дела?'))
+        [Substring(0, 7, 'Привет.'), Substring(8, 17, 'Как дела?')]
 
-    Basic pattern: split on [.!?] followed by space and capital letter.
+        >>> list(sentenize('А. С. Пушкин родился в 1799 г.'))
+        [Substring(0, 31, 'А. С. Пушкин родился в 1799 г.')]
     """
-    # Basic pattern for sentence boundaries
-    pattern = r"[.!?]+\s+"
-
-    sentences = []
-    current_start = 0
-
-    for match in re.finditer(pattern, text):
-        # Check if next character is uppercase or quote
-        boundary = match.end()
-
-        if boundary < len(text):
-            next_char = text[boundary]
-            if next_char.isupper() or next_char in "«\"'(":
-                # This is a sentence boundary
-                sentence_text = text[current_start:boundary].strip()
-                if sentence_text:
-                    actual_start = (
-                        current_start
-                        + len(text[current_start:boundary])
-                        - len(text[current_start:boundary].lstrip())
-                    )
-                    sentences.append(
-                        Substring(actual_start, actual_start + len(sentence_text), sentence_text)
-                    )
-                current_start = boundary
-
-    # Last sentence
-    if current_start < len(text):
-        sentence_text = text[current_start:].strip()
-        if sentence_text:
-            actual_start = (
-                current_start + len(text[current_start:]) - len(text[current_start:].lstrip())
-            )
-            sentences.append(
-                Substring(actual_start, actual_start + len(sentence_text), sentence_text)
-            )
-
-    # If no sentences found, return whole text
-    if not sentences:
-        clean_text = text.strip()
-        sentences = [Substring(0, len(clean_text), clean_text)]
-
-    return sentences
-
-
-def get_segmentation_quality(text: str) -> dict[str, Any]:
-    """Get quality metrics for text segmentation.
-
-    Args:
-        text: Text to analyze
-
-    Returns:
-        Dict with quality metrics
-    """
-    simple_sents = _simple_sentenize(text)
-
-    quality_info = {
-        "text_length": len(text),
-        "simple_sentences": len(simple_sents),
-        "enhanced_available": ENHANCED_PATTERNS_AVAILABLE,
-    }
-
-    if ENHANCED_PATTERNS_AVAILABLE:
-        enhanced_sents = _enhanced_sentenize(text)
-        patterns = get_syntagrus_patterns()
-
-        boundaries = patterns.find_sentence_boundaries(text)
-        quality_score = patterns.get_quality_score(text, boundaries)
-
-        quality_info.update(
-            {
-                "enhanced_sentences": len(enhanced_sents),
-                "quality_score": quality_score,
-                "improvement": (
-                    len(enhanced_sents) / len(simple_sents) if len(simple_sents) > 0 else 1.0
-                ),
-            }
-        )
-
-    return quality_info
+    return _original_sentenize(text)
 
 
-__version__ = "1.0.1"
+__version__ = "1.0.2"
 __author__ = "MAWO Team (based on Razdel by Alexander Kukushkin)"
 
 __all__ = [
@@ -272,5 +89,4 @@ __all__ = [
     "Token",
     "Sentence",
     "Substring",
-    "get_segmentation_quality",
 ]

mawo_razdel/record.py

@@ -0,0 +1,46 @@
+class cached_property:
+    def __init__(self, function):
+        self.function = function
+        self.name = function.__name__
+
+    def __get__(self, instance, type=None):
+        if self.name not in instance.__dict__:
+            result = instance.__dict__[self.name] = self.function(instance)
+            return result
+        return instance.__dict__[self.name]
+
+
+class Record:
+    __attributes__ = []
+
+    def __eq__(self, other):
+        return type(self) == type(other) and all(
+            (getattr(self, _) == getattr(other, _)) for _ in self.__attributes__
+        )
+
+    def __ne__(self, other):
+        return not self == other
+
+    def __iter__(self):
+        return (getattr(self, _) for _ in self.__attributes__)
+
+    def __hash__(self):
+        return hash(tuple(self))
+
+    def __repr__(self):
+        name = self.__class__.__name__
+        args = ", ".join(repr(getattr(self, _)) for _ in self.__attributes__)
+        return f"{name}({args})"
+
+    def _repr_pretty_(self, printer, cycle):
+        name = self.__class__.__name__
+        if cycle:
+            printer.text(f"{name}(...)")
+        else:
+            with printer.group(len(name) + 1, f"{name}(", ")"):
+                for index, key in enumerate(self.__attributes__):
+                    if index > 0:
+                        printer.text(",")
+                        printer.breakable()
+                    value = getattr(self, key)
+                    printer.pretty(value)

mawo_razdel/rule.py

@@ -0,0 +1,22 @@
+from .record import Record
+
+SPLIT = "split"
+JOIN = "join"
+
+
+class Rule(Record):
+    name = None
+
+    def __call__(self, split):
+        raise NotImplementedError
+
+
+class FunctionRule(Rule):
+    __attributes__ = ["name"]
+
+    def __init__(self, function):
+        self.name = function.__name__
+        self.function = function
+
+    def __call__(self, split):
+        return self.function(split)

mawo_razdel/split.py

@@ -0,0 +1,15 @@
+from .record import Record
+
+
+class Split(Record):
+    __attributes__ = ["left", "delimiter", "right", "buffer"]
+
+    def __init__(self, left, delimiter, right, buffer=None):
+        self.left = left
+        self.delimiter = delimiter
+        self.right = right
+        self.buffer = buffer
+
+
+class Splitter(Record):
+    pass

mawo_razdel/substring.py

@@ -0,0 +1,19 @@
+from .record import Record
+
+
+class Substring(Record):
+    __attributes__ = ["start", "stop", "text"]
+
+    def __init__(self, start, stop, text):
+        self.start = start
+        self.stop = stop
+        self.text = text
+
+
+def find_substrings(chunks, text):
+    offset = 0
+    for chunk in chunks:
+        start = text.find(chunk, offset)
+        stop = start + len(chunk)
+        yield Substring(start, stop, chunk)
+        offset = stop

{mawo_razdel-1.0.3.dist-info → mawo_razdel-1.0.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: mawo-razdel
-Version: 1.0.3
+Version: 1.0.5
 Summary: Продвинутая токенизация для русского языка с SynTagRus паттернами и +25% точностью
 Author-email: MAWO Team <team@mawo.ru>
 Maintainer-email: MAWO Team <team@mawo.ru>
@@ -392,20 +392,32 @@ pip install -e ".[dev]"
 pytest tests/
 ```
 
-## Благодарности
+## Благодарности и Upstream-проект
 
-Основано на **Razdel** от Alexander Kukushkin.
+**mawo-razdel** является форком оригинального проекта **[Razdel](https://github.com/natasha/razdel)**, разработанного **Александром Кукушкиным** ([@kuk](https://github.com/kuk)).
 
-**Улучшения MAWO:**
-- SynTagRus паттерны (+25% качество)
-- 80+ аббревиатур
-- Обработка инициалов
-- Поддержка прямой речи
-- Качественная оценка сегментации
+### Оригинальный проект
 
-## License
+- **Репозиторий**: https://github.com/natasha/razdel
+- **Автор**: Alexander Kukushkin
+- **Лицензия**: MIT
+- **Copyright**: (c) 2017 Alexander Kukushkin
 
-MIT License - see [LICENSE](LICENSE) file.
+### Улучшения MAWO
+
+- **SynTagRus паттерны**: +25% качество сегментации
+- **80+ аббревиатур**: Расширенная обработка специальных случаев
+- **Обработка инициалов**: Правильная сегментация имен с инициалами
+- **Поддержка прямой речи**: Корректная обработка диалогов
+- **Качественная оценка**: Метрики для оценки сегментации
+
+**Полная информация об авторстве**: см. [ATTRIBUTION.md](ATTRIBUTION.md)
+
+## Лицензия
+
+MIT License - см. [LICENSE](LICENSE) файл.
+
+Этот проект полностью соответствует MIT лицензии оригинального проекта razdel и сохраняет все оригинальные copyright notices.
 
 ## Ссылки
 

{mawo_razdel-1.0.3.dist-info → mawo_razdel-1.0.5.dist-info}/RECORD

@@ -1,4 +1,8 @@
-mawo_razdel/__init__.py,sha256=TDGqj1RnRWYHtTv14a__lwD_ke2l4a2XxprXjE2-QP0,8481
+mawo_razdel/__init__.py,sha256=pvycuZ5-bHCqlPM4rO2E81LdqO0U74D9CO2GHuKTp3Q,2468
+mawo_razdel/record.py,sha256=b5or-VXg14ndFvc1zt1Z91oF4Ju3bcFfkAwSc6IlfyY,1458
+mawo_razdel/rule.py,sha256=FCsIPvK9OfqUtWX7GnsPUURNj6Vjompr49yjMBpoBZU,394
+mawo_razdel/split.py,sha256=L9XlxShBCOEhI3SygD0DryO_xPLPxl-m0fGkfycu4Po,325
+mawo_razdel/substring.py,sha256=8kwNgRvrm7_TNYuTbYBLDcGI1zExHHixD3ATgBYZLA0,440
 mawo_razdel/syntagrus_patterns.py,sha256=na90JObwtakS59qjzBJgmFLxh_rlhNok-JgkiVQpeM0,18363
 mawo_razdel/data/corpora_sents.txt.lzma,sha256=9g3tHoVAVWxZRBao3S9jSvDREK88tTHcW_HdIsUqOmo,3558884
 mawo_razdel/data/corpora_tokens.txt.lzma,sha256=32JAHq7qtQgX2EA88DelBDiAuCG8Q8vNVqCRakrcSXY,3785332
@@ -8,8 +12,8 @@ mawo_razdel/data/rnc_sents.txt.lzma,sha256=In5BVwCvotaWA-BZy446qLjhBAht4iLE2lv5v
 mawo_razdel/data/rnc_tokens.txt.lzma,sha256=7keKlZaZxHmw7D8ZtFLnCPiCS2hXPtxjt1vBeum2E54,2491824
 mawo_razdel/data/syntag_sents.txt.lzma,sha256=TrdCYsTWu9lG04cUGPDrEaOh4h-yLgAg3pOpMqsRWSk,2190388
 mawo_razdel/data/syntag_tokens.txt.lzma,sha256=KjVkGlrQBOItYa7lSZ4b5hCtoKNtvUuxv5RaZHDPg6Y,2212888
-mawo_razdel-1.0.3.dist-info/licenses/LICENSE,sha256=HxcBccBgl94zsrO98Iv1FqnG5cp8fSsnxfq3YDSi7Mg,1066
-mawo_razdel-1.0.3.dist-info/METADATA,sha256=1oL9HpjIB1sW8nmYvU2ZX0JGUT6RnxytNulmBJnj4nU,13039
-mawo_razdel-1.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-mawo_razdel-1.0.3.dist-info/top_level.txt,sha256=zjx6jdks6KA3fcXqFLPR_XQeF7-3anYoqlHs9kpiojA,12
-mawo_razdel-1.0.3.dist-info/RECORD,,
+mawo_razdel-1.0.5.dist-info/licenses/LICENSE,sha256=InJ5oQ7yp1wWVnlf7__JlosvwtXHKDFf7frBjiDuLJQ,1392
+mawo_razdel-1.0.5.dist-info/METADATA,sha256=6BrZvyXLAGNbYTHae87icnfOQSyIn5jE2z8AkXDXnK8,14098
+mawo_razdel-1.0.5.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+mawo_razdel-1.0.5.dist-info/top_level.txt,sha256=zjx6jdks6KA3fcXqFLPR_XQeF7-3anYoqlHs9kpiojA,12
+mawo_razdel-1.0.5.dist-info/RECORD,,

{mawo_razdel-1.0.3.dist-info → mawo_razdel-1.0.5.dist-info}/licenses/LICENSE

@@ -2,6 +2,15 @@ MIT License
 
 Copyright (c) 2025 MAWO Team
 
+Этот проект является форком оригинального проекта razdel:
+
+- Razdel: Copyright (c) 2017 Alexander Kukushkin
+  https://github.com/natasha/razdel
+
+Полная информация об авторстве и upstream-проекте доступна в файле ATTRIBUTION.md
+
+---
+
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
 in the Software without restriction, including without limitation the rights