sharedkernel 2.7.0__tar.gz → 2.8.0__tar.gz

{sharedkernel-2.7.0 → sharedkernel-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.7.0
+Version: 2.8.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -31,6 +31,8 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
 ### Version 2.7.0
 - Add Redis and cache repository
 ### Version 2.6.4

{sharedkernel-2.7.0 → sharedkernel-2.8.0}/README.md

@@ -2,6 +2,8 @@
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
 ### Version 2.7.0
 - Add Redis and cache repository
 ### Version 2.6.4

{sharedkernel-2.7.0 → sharedkernel-2.8.0}/setup.py

@@ -19,6 +19,8 @@ setup(
         "sharedkernel.objects",
         "sharedkernel.normalizer",
         "sharedkernel.logger",
+        "sharedkernel.chunker",
+        
     ],
     # Needed for dependencies
     install_requires=[
@@ -41,7 +43,7 @@ setup(
         "redis==8.0.0",
     ],
     # *strongly* suggested for sharing
-    version="2.7.0",
+    version="2.8.0",
     description="sharekernel is a shared package between all python projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

sharedkernel-2.8.0/sharedkernel/chunker/chunk_rule.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+import re
+from typing import List, Optional, Protocol, Tuple
+
+
+class SplitRule(Protocol):
+    """Interface for chunk split rules."""
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]: ...
+
+
+class PunctuationRule(SplitRule):
+    """Split after the first matching punctuation character."""
+
+    def __init__(self, punctuations: str = ".,;:!?"):
+        self.punctuations = set(punctuations)
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        for i, ch in enumerate(segment):
+            if ch in self.punctuations:
+                return offset + i + 1, ch, "punctuation"
+        return None, None, None
+
+
+class RegexRule(SplitRule):
+    """Split after the first regex match."""
+
+    def __init__(self, pattern: str):
+        self.regex = re.compile(pattern) if pattern else None
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        if not self.regex:
+            return None, None, None
+        m = self.regex.search(segment)
+        if m:
+            return offset + m.end(), segment[m.start() : m.end()], "regex"
+        return None, None, None
+
+
+class CutWordRule(SplitRule):
+    """Split after the first occurrence of any configured word/phrase."""
+
+    def __init__(self, cut_words: List[str]):
+        self.cut_words = [w.lower() for w in cut_words]
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        lowered = segment.lower()
+        for word in self.cut_words:
+            idx = lowered.find(word)
+            if idx != -1:
+                phrase = segment[idx : idx + len(word)]
+                return offset + idx + len(word), phrase, "cut_word"
+        return None, None, None
+
+
+class SpaceFallbackRule(SplitRule):
+    """Split near the center on space; otherwise hard-cut at max_len."""
+
+    def __init__(self, max_len: int):
+        self.max_len = max_len
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        centre = len(segment) // 2
+        left = segment.rfind(" ", 0, centre)
+        right = segment.find(" ", centre)
+
+        if left != -1:
+            split = left
+        elif right != -1:
+            split = right
+        else:
+            split = self.max_len
+
+        return offset + split, None, "space_fallback"

sharedkernel-2.8.0/sharedkernel/chunker/text_chunker.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from .chunk_rule import (
+    SplitRule,
+)
+
+
+@dataclass
+class Chunk:
+    """A single chunk produced by TextChunker."""
+
+    text: str
+    number: int
+    begin: int
+    end: int
+    method: str
+    phrase: Optional[str]
+
+
+class TextChunker:
+    """Split text into chunks based on ordered split rules."""
+
+    def __init__(self, min_len: int, max_len: int, rules: List[SplitRule]):
+        if min_len < 1:
+            raise ValueError("min_len must be >= 1")
+        if max_len < min_len:
+            raise ValueError("max_len must be >= min_len")
+
+        self.min_len = min_len
+        self.max_len = max_len
+        self.rules = rules
+
+    def chunk(self, text: str) -> List[Chunk]:
+        if not text:
+            return []
+
+        chunks: List[Chunk] = []
+        pos = 0
+        number = 1
+
+        while pos < len(text):
+            if len(text) - pos <= self.max_len:
+                chunk_text = text[pos:]
+                chunks.append(Chunk(chunk_text, number, pos, len(text), "final", None))
+                break
+
+            window_end = pos + self.max_len
+            window = text[pos:window_end]
+
+            split_idx: Optional[int] = None
+            phrase: Optional[str] = None
+            method: Optional[str] = None
+
+            for rule in self.rules:
+                candidate, cand_phrase, cand_method = rule.apply(window, pos)
+                if candidate is not None and candidate - pos >= self.min_len:
+                    split_idx, phrase, method = candidate, cand_phrase, cand_method
+                    break
+
+            if split_idx is None:
+                split_idx = pos + self.max_len
+                method = "hard_cut"
+
+            chunk_text = text[pos:split_idx]
+            chunks.append(Chunk(chunk_text, number, pos, split_idx, method, phrase))
+            pos = split_idx
+            number += 1
+
+        return chunks
+
+    def chunks(self, text: str):
+        """Yield chunks lazily, using the same logic as `chunk`."""
+        if not text:
+            return
+
+        pos = 0
+        number = 1
+
+        while pos < len(text):
+            if len(text) - pos <= self.max_len:
+                chunk_text = text[pos:]
+                yield Chunk(chunk_text, number, pos, len(text), "final", None)
+                break
+
+            window_end = pos + self.max_len
+            window = text[pos:window_end]
+
+            split_idx: Optional[int] = None
+            phrase: Optional[str] = None
+            method: Optional[str] = None
+
+            for rule in self.rules:
+                candidate, cand_phrase, cand_method = rule.apply(window, pos)
+                if candidate is not None and candidate - pos >= self.min_len:
+                    split_idx, phrase, method = candidate, cand_phrase, cand_method
+                    break
+
+            if split_idx is None:
+                split_idx = pos + self.max_len
+                method = "hard_cut"
+
+            chunk_text = text[pos:split_idx]
+            yield Chunk(chunk_text, number, pos, split_idx, method, phrase)
+
+            pos = split_idx
+            number += 1

{sharedkernel-2.7.0 → sharedkernel-2.8.0}/sharedkernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.7.0
+Version: 2.8.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -31,6 +31,8 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
 ### Version 2.7.0
 - Add Redis and cache repository
 ### Version 2.6.4

{sharedkernel-2.7.0 → sharedkernel-2.8.0}/sharedkernel.egg-info/SOURCES.txt

@@ -17,6 +17,8 @@ sharedkernel.egg-info/SOURCES.txt
 sharedkernel.egg-info/dependency_links.txt
 sharedkernel.egg-info/requires.txt
 sharedkernel.egg-info/top_level.txt
+sharedkernel/chunker/chunk_rule.py
+sharedkernel/chunker/text_chunker.py
 sharedkernel/database/__init__.py
 sharedkernel/database/audit_model.py
 sharedkernel/database/distributed_cache.py