sharedkernel 2.6.4__tar.gz → 2.8.0__tar.gz

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.6.4
+Version: 2.8.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -20,7 +20,7 @@ Requires-Dist: markdown
 Requires-Dist: beautifulsoup4
 Requires-Dist: deepdiff
 Requires-Dist: kombu
-Requires-Dist: redis
+Requires-Dist: redis==8.0.0
 Dynamic: author
 Dynamic: description
 Dynamic: description-content-type
@@ -31,6 +31,10 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
+### Version 2.7.0
+- Add Redis and cache repository
 ### Version 2.6.4
 - Add Verbose to log
 ### Version 2.6.3

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/README.md

@@ -2,6 +2,10 @@
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
+### Version 2.7.0
+- Add Redis and cache repository
 ### Version 2.6.4
 - Add Verbose to log
 ### Version 2.6.3

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/setup.py

@@ -13,11 +13,14 @@ setup(
     packages=[
         "sharedkernel",
         "sharedkernel.database",
+        "sharedkernel.database.cache",
         "sharedkernel.enum",
         "sharedkernel.exception",
         "sharedkernel.objects",
         "sharedkernel.normalizer",
         "sharedkernel.logger",
+        "sharedkernel.chunker",
+        
     ],
     # Needed for dependencies
     install_requires=[
@@ -37,10 +40,10 @@ setup(
         "beautifulsoup4",
         "deepdiff",
         "kombu",
-        "redis",
+        "redis==8.0.0",
     ],
     # *strongly* suggested for sharing
-    version="2.6.4",
+    version="2.8.0",
     description="sharekernel is a shared package between all python projects",
     long_description=long_description,
     long_description_content_type="text/markdown",

sharedkernel-2.8.0/sharedkernel/chunker/chunk_rule.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+import re
+from typing import List, Optional, Protocol, Tuple
+
+
+class SplitRule(Protocol):
+    """Interface for chunk split rules."""
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]: ...
+
+
+class PunctuationRule(SplitRule):
+    """Split after the first matching punctuation character."""
+
+    def __init__(self, punctuations: str = ".,;:!?"):
+        self.punctuations = set(punctuations)
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        for i, ch in enumerate(segment):
+            if ch in self.punctuations:
+                return offset + i + 1, ch, "punctuation"
+        return None, None, None
+
+
+class RegexRule(SplitRule):
+    """Split after the first regex match."""
+
+    def __init__(self, pattern: str):
+        self.regex = re.compile(pattern) if pattern else None
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        if not self.regex:
+            return None, None, None
+        m = self.regex.search(segment)
+        if m:
+            return offset + m.end(), segment[m.start() : m.end()], "regex"
+        return None, None, None
+
+
+class CutWordRule(SplitRule):
+    """Split after the first occurrence of any configured word/phrase."""
+
+    def __init__(self, cut_words: List[str]):
+        self.cut_words = [w.lower() for w in cut_words]
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        lowered = segment.lower()
+        for word in self.cut_words:
+            idx = lowered.find(word)
+            if idx != -1:
+                phrase = segment[idx : idx + len(word)]
+                return offset + idx + len(word), phrase, "cut_word"
+        return None, None, None
+
+
+class SpaceFallbackRule(SplitRule):
+    """Split near the center on space; otherwise hard-cut at max_len."""
+
+    def __init__(self, max_len: int):
+        self.max_len = max_len
+
+    def apply(
+        self, segment: str, offset: int
+    ) -> Tuple[Optional[int], Optional[str], Optional[str]]:
+        centre = len(segment) // 2
+        left = segment.rfind(" ", 0, centre)
+        right = segment.find(" ", centre)
+
+        if left != -1:
+            split = left
+        elif right != -1:
+            split = right
+        else:
+            split = self.max_len
+
+        return offset + split, None, "space_fallback"

sharedkernel-2.8.0/sharedkernel/chunker/text_chunker.py

@@ -0,0 +1,107 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from .chunk_rule import (
+    SplitRule,
+)
+
+
+@dataclass
+class Chunk:
+    """A single chunk produced by TextChunker."""
+
+    text: str
+    number: int
+    begin: int
+    end: int
+    method: str
+    phrase: Optional[str]
+
+
+class TextChunker:
+    """Split text into chunks based on ordered split rules."""
+
+    def __init__(self, min_len: int, max_len: int, rules: List[SplitRule]):
+        if min_len < 1:
+            raise ValueError("min_len must be >= 1")
+        if max_len < min_len:
+            raise ValueError("max_len must be >= min_len")
+
+        self.min_len = min_len
+        self.max_len = max_len
+        self.rules = rules
+
+    def chunk(self, text: str) -> List[Chunk]:
+        if not text:
+            return []
+
+        chunks: List[Chunk] = []
+        pos = 0
+        number = 1
+
+        while pos < len(text):
+            if len(text) - pos <= self.max_len:
+                chunk_text = text[pos:]
+                chunks.append(Chunk(chunk_text, number, pos, len(text), "final", None))
+                break
+
+            window_end = pos + self.max_len
+            window = text[pos:window_end]
+
+            split_idx: Optional[int] = None
+            phrase: Optional[str] = None
+            method: Optional[str] = None
+
+            for rule in self.rules:
+                candidate, cand_phrase, cand_method = rule.apply(window, pos)
+                if candidate is not None and candidate - pos >= self.min_len:
+                    split_idx, phrase, method = candidate, cand_phrase, cand_method
+                    break
+
+            if split_idx is None:
+                split_idx = pos + self.max_len
+                method = "hard_cut"
+
+            chunk_text = text[pos:split_idx]
+            chunks.append(Chunk(chunk_text, number, pos, split_idx, method, phrase))
+            pos = split_idx
+            number += 1
+
+        return chunks
+
+    def chunks(self, text: str):
+        """Yield chunks lazily, using the same logic as `chunk`."""
+        if not text:
+            return
+
+        pos = 0
+        number = 1
+
+        while pos < len(text):
+            if len(text) - pos <= self.max_len:
+                chunk_text = text[pos:]
+                yield Chunk(chunk_text, number, pos, len(text), "final", None)
+                break
+
+            window_end = pos + self.max_len
+            window = text[pos:window_end]
+
+            split_idx: Optional[int] = None
+            phrase: Optional[str] = None
+            method: Optional[str] = None
+
+            for rule in self.rules:
+                candidate, cand_phrase, cand_method = rule.apply(window, pos)
+                if candidate is not None and candidate - pos >= self.min_len:
+                    split_idx, phrase, method = candidate, cand_phrase, cand_method
+                    break
+
+            if split_idx is None:
+                split_idx = pos + self.max_len
+                method = "hard_cut"
+
+            chunk_text = text[pos:split_idx]
+            yield Chunk(chunk_text, number, pos, split_idx, method, phrase)
+
+            pos = split_idx
+            number += 1

sharedkernel-2.8.0/sharedkernel/database/cache/cache_repository.py

@@ -0,0 +1,37 @@
+
+from abc import ABC, abstractmethod
+from typing import Generic, Optional, TypeVar
+from pydantic import BaseModel
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class CacheRepository(ABC, Generic[T]):
+    @abstractmethod
+    async def set(
+        self,
+        key: str,
+        value: T,
+        ttl_seconds: int | None = None,
+    ) -> None:
+        pass
+
+    @abstractmethod
+    async def get(self, key: str) -> Optional[T]:
+        pass
+
+    @abstractmethod
+    async def exists(self, key: str) -> bool:
+        pass
+
+    @abstractmethod
+    async def delete(self, key: str) -> int:
+        pass
+
+    @abstractmethod
+    async def delete_group(self, key_prefix: str) -> int:
+        pass
+
+    @abstractmethod
+    async def clear(self) -> int:
+        pass

sharedkernel-2.8.0/sharedkernel/database/cache/redis_generic_cache.py

@@ -0,0 +1,88 @@
+from typing import Generic, Optional, Type, TypeVar
+from pydantic import BaseModel
+from redis.asyncio import Redis
+
+from cache.cache_repository import CacheRepository
+
+T = TypeVar("T", bound=BaseModel)
+
+
+class RedisGenericCache(CacheRepository[T], Generic[T]):
+    def __init__(
+        self,
+        client: Redis,
+        namespace: str,
+        model: Type[T],
+        default_ttl_seconds: int | None = None,
+    ):
+        self.client = client
+        self.namespace = namespace
+        self.model = model
+        self.default_ttl_seconds = default_ttl_seconds
+
+    def _pref(self, key: str) -> str:
+        return f"{self.namespace}:{key}"
+
+    def _unpref(self, key: str) -> str:
+        prefix = f"{self.namespace}:"
+        return key.removeprefix(prefix)
+
+    def _serialize(self, value: T) -> str:
+        return value.model_dump_json()
+
+    def _deserialize(self, raw: str | bytes) -> T:
+        return self.model.model_validate_json(raw.decode())
+
+    async def set(
+        self,
+        key: str,
+        value: T,
+        ttl_seconds: int | None = None,
+    ) -> None:
+        if value is None:
+            return
+
+        redis_key = self._pref(key)
+        ttl = ttl_seconds if ttl_seconds else self.default_ttl_seconds
+
+        if ttl:
+            await self.client.set(redis_key, self._serialize(value), ex=ttl)
+        else:
+            await self.client.set(redis_key, self._serialize(value))
+
+    async def get(self, key: str) -> Optional[T]:
+        raw = await self.client.get(self._pref(key))
+        if raw is None:
+            return None
+        return self._deserialize(raw)
+
+    async def exists(self, key: str) -> bool:
+        return bool(await self.client.exists(self._pref(key)))
+
+    async def delete(self, key: str) -> int:
+        return await self.client.delete(self._pref(key))
+
+    async def delete_group(self, key_prefix: str) -> int:
+        pattern = self._pref(f"{key_prefix}*")
+        deleted = 0
+
+        async for key in self.client.scan_iter(match=pattern):
+            deleted += await self.client.delete(key)
+
+        return deleted
+
+    async def clear(self) -> int:
+        deleted = 0
+
+        async for key in self.client.scan_iter(match=f"{self.namespace}:*"):
+            deleted += await self.client.delete(key)
+
+        return deleted
+
+    async def get_all_ids(self) -> list[str]:
+        result: list[str] = []
+
+        async for raw in self.client.scan_iter(match=f"{self.namespace}:*"):
+            result.append(self._unpref(raw.decode()))
+
+        return result

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/sharedkernel.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: sharedkernel
-Version: 2.6.4
+Version: 2.8.0
 Summary: sharekernel is a shared package between all python projects
 Author: Smilinno
 Description-Content-Type: text/markdown
@@ -20,7 +20,7 @@ Requires-Dist: markdown
 Requires-Dist: beautifulsoup4
 Requires-Dist: deepdiff
 Requires-Dist: kombu
-Requires-Dist: redis
+Requires-Dist: redis==8.0.0
 Dynamic: author
 Dynamic: description
 Dynamic: description-content-type
@@ -31,6 +31,10 @@ Dynamic: summary
 this is a shared kernel package
 
 # Change Log
+### Version 2.8.0
+- Add chunker for tts
+### Version 2.7.0
+- Add Redis and cache repository
 ### Version 2.6.4
 - Add Verbose to log
 ### Version 2.6.3

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/sharedkernel.egg-info/SOURCES.txt

@@ -17,6 +17,8 @@ sharedkernel.egg-info/SOURCES.txt
 sharedkernel.egg-info/dependency_links.txt
 sharedkernel.egg-info/requires.txt
 sharedkernel.egg-info/top_level.txt
+sharedkernel/chunker/chunk_rule.py
+sharedkernel/chunker/text_chunker.py
 sharedkernel/database/__init__.py
 sharedkernel/database/audit_model.py
 sharedkernel/database/distributed_cache.py
@@ -24,6 +26,9 @@ sharedkernel/database/mongo_generic_audit_repository.py
 sharedkernel/database/mongo_generic_repository.py
 sharedkernel/database/mongo_health_checker.py
 sharedkernel/database/pagination_response_dto.py
+sharedkernel/database/cache/__init__.py
+sharedkernel/database/cache/cache_repository.py
+sharedkernel/database/cache/redis_generic_cache.py
 sharedkernel/enum/__init__.py
 sharedkernel/enum/error_code.py
 sharedkernel/enum/redis_mode_enum.py

{sharedkernel-2.6.4 → sharedkernel-2.8.0}/sharedkernel.egg-info/requires.txt

@@ -14,4 +14,4 @@ markdown
 beautifulsoup4
 deepdiff
 kombu
-redis
+redis==8.0.0