toolkitx 0.0.1__py3-none-any.whl

toolkitx/__init__.py

@@ -0,0 +1,13 @@
+# 从子模块导入，使其可以直接从 toolkitx 包导入
+from .hello import hello
+from .text_utils import truncate_text_smart, split_text_by_word_count
+
+# 定义 __all__，当用户使用 from toolkitx import * 时，会导入这些符号
+__all__ = [
+    "hello",
+    "truncate_text_smart",
+    "split_text_by_word_count"
+]
+
+# 你也可以在这里定义包级别的版本号等元数据
+__version__ = "0.0.1" # 与 pyproject.toml 中的 version 一致

toolkitx/hello.py

@@ -0,0 +1,2 @@
+def hello():
+    print("hello! I'am toolkitx")

toolkitx/lab/translator.py

@@ -0,0 +1,230 @@
+import os
+from abc import ABC, abstractmethod
+from typing import Optional, Literal
+from diskcache import Cache
+from transgpt.trans_baidu import BaiduTranslation
+from transgpt.trans_tencent import TencentTranslation
+import hashlib
+
+# 定义支持的翻译引擎类型
+SUPPORTED_ENGINES = Literal["baidu", "tencent"]
+
+class TranslatorInterface(ABC):
+    """
+    Abstract base class for a translator.
+    Defines the common interface for translation services.
+    """
+
+    @abstractmethod
+    def translate(self, text: str, target_lang: str, source_lang: str = 'auto') -> str:
+        """
+        Translate text from a source language to a target language.
+
+        :param text: The text to translate.
+        :param target_lang: The target language code (e.g., 'en', 'zh').
+        :param source_lang: The source language code (e.g., 'auto', 'en', 'zh').
+        :return: The translated text.
+        """
+        pass
+
+    @abstractmethod
+    def clear_cache(self) -> None:
+        """
+        Clear the translation cache.
+        """
+        pass
+
+
+class Translator(TranslatorInterface):
+    """
+    A translator implementation using py-transgpt with disk-based caching.
+    Supports Baidu and Tencent translation engines.
+    """
+
+    def __init__(
+        self,
+        engine: SUPPORTED_ENGINES,
+        cache_path: str,
+        api_id: Optional[str] = None,
+        api_key: Optional[str] = None,
+        target_lang: str = 'en', # Default target language
+        source_lang: str = 'auto' # Default source language
+    ):
+        """
+        Initialize the Translator.
+
+        :param engine: The translation engine to use ('baidu' or 'tencent').
+        :param cache_path: Path to the directory for storing the translation cache.
+        :param api_id: API ID for the translation service. If None, attempts to load from ENV.
+                       For Tencent, this is SecretId.
+        :param api_key: API Key for the translation service. If None, attempts to load from ENV.
+                        For Tencent, this is SecretKey.
+        :param target_lang: Default target language for translations.
+        :param source_lang: Default source language for translations.
+        :raises ValueError: If the engine is not supported or API credentials are not found.
+        """
+        if engine not in ["baidu", "tencent"]:
+            raise ValueError(f"Unsupported engine: {engine}. Supported engines are 'baidu', 'tencent'.")
+
+        self.engine_name = engine
+        self.cache = Cache(cache_path)
+        self.default_target_lang = target_lang
+        self.default_source_lang = source_lang
+
+        # Load API credentials
+        env_api_id_name = ""
+        env_api_key_name = ""
+
+        if self.engine_name == "baidu":
+            env_api_id_name = "BAIDU_API_ID"
+            env_api_key_name = "BAIDU_API_KEY"
+            _api_id = api_id or os.getenv(env_api_id_name)
+            _api_key = api_key or os.getenv(env_api_key_name)
+            if not _api_id or not _api_key:
+                raise ValueError(
+                    f"Baidu API ID and API Key are required. "
+                    f"Provide them as arguments or set {env_api_id_name} and {env_api_key_name} environment variables."
+                )
+            self.translator_instance = BaiduTranslation(api_id=_api_id, api_key=_api_key)
+
+        elif self.engine_name == "tencent":
+            env_api_id_name = "TENCENT_API_ID"
+            env_api_key_name = "TENCENT_API_KEY"
+            _api_id = api_id or os.getenv(env_api_id_name)
+            _api_key = api_key or os.getenv(env_api_key_name)
+            if not _api_id or not _api_key:
+                raise ValueError(
+                    f"Tencent Secret ID and Secret Key are required. "
+                    f"Provide them as arguments or set {env_api_id_name} and {env_api_key_name} environment variables."
+                )
+            self.translator_instance = TencentTranslation(api_id=_api_id, api_key=_api_key)
+        
+    def _create_cache_key(self, text: str, target_lang: str, source_lang: str) -> str:
+        """Helper to create a unique cache key."""
+        return f"{self.engine_name}:{source_lang}:{target_lang}:{hashlib.md5(text.encode("utf8")).hexdigest()}"
+
+    def translate(self, text: str, target_lang: Optional[str] = None, source_lang: Optional[str] = None) -> str:
+        """
+        Translate text using the configured engine and cache.
+
+        :param text: The text to translate.
+        :param target_lang: The target language code. Defaults to instance's default_target_lang.
+        :param source_lang: The source language code. Defaults to instance's default_source_lang.
+        :return: The translated text.
+        """
+        _target_lang = target_lang or self.default_target_lang
+        _source_lang = source_lang or self.default_source_lang
+
+        if not text:
+            return ""
+
+        cache_key = self._create_cache_key(text, _target_lang, _source_lang)
+        cached_translation = self.cache.get(cache_key)
+
+        if cached_translation is not None:
+            return cached_translation
+        print("no cache, cache_key:",cache_key)
+        # print(f"Cache miss. Translating: {text[:30]}...") # For debugging
+        try:
+            # py-transgpt's translate method takes (text, target_language, source_language)
+            translated_text = self.translator_instance.translate(text, _target_lang, _source_lang)
+            print("set cache:",text,translated_text)
+            if translated_text: # Ensure we don't cache None or empty if translation fails silently
+                self.cache.set(cache_key, translated_text)
+            return translated_text
+        except Exception as e:
+            # Log error or handle as needed
+            print(f"Error during translation with {self.engine_name}: {e}")
+            # Depending on requirements, you might want to re-raise or return original text/error message
+            raise  # Re-raise the exception to make the caller aware
+
+    def clear_cache(self) -> None:
+        """
+        Clear all items from the translation cache for this translator instance.
+        """
+        self.cache.clear()
+        print(f"Cache cleared for path: {self.cache.directory}")
+
+    def close_cache(self) -> None:
+        """
+        Close the cache. Important to call when done if cache is not used as a context manager.
+        """
+        self.cache.close()
+
+# Example Usage (you can put this in a separate test file or a __main__ block)
+if __name__ == "__main__":
+    # Ensure you have set your environment variables for BAIDU_API_ID, BAIDU_API_KEY
+    # or TENCENT_SECRET_ID, TENCENT_SECRET_KEY, or pass them directly.
+
+    # Example for Baidu (assuming ENV VARS are set)
+    try:
+        print("Testing Baidu Translator...")
+        # Create a directory for cache if it doesn't exist
+        baidu_cache_dir = "/tmp/translator_cache/baidu"
+        os.makedirs(baidu_cache_dir, exist_ok=True)
+
+        baidu_translator = Translator(engine="baidu", cache_path=baidu_cache_dir, target_lang='en', source_lang='zh')
+        
+        text_to_translate_zh = "你好，世界！"
+        print(f"Original (zh): {text_to_translate_zh}")
+
+        # First translation (should be from API)
+        translated_en = baidu_translator.translate(text_to_translate_zh)
+        print(f"Translated (en): {translated_en}")
+
+        # Second translation (should be from cache)
+        translated_en_cached = baidu_translator.translate(text_to_translate_zh)
+        print(f"Translated (en) from cache: {translated_en_cached}")
+        
+        assert translated_en == translated_en_cached
+
+        # Translate to French
+        translated_fr = baidu_translator.translate(text_to_translate_zh, target_lang='fr') # Baidu uses 'fra' for French
+        print(f"Translated (fr): {translated_fr}")
+
+
+        # baidu_translator.clear_cache()
+        baidu_translator.close_cache()
+        print("Baidu Translator test finished.\n")
+
+    except ValueError as ve:
+        print(f"Skipping Baidu test: {ve}")
+    except Exception as e:
+        print(f"Error during Baidu test: {e}")
+
+    # Example for Tencent (assuming ENV VARS are set)
+    try:
+        print("Testing Tencent Translator...")
+        tencent_cache_dir = "/tmp/translator_cache/tencent"
+        os.makedirs(tencent_cache_dir, exist_ok=True)
+        
+        # You can pass api_id and api_key directly if not using ENV VARS
+        # tencent_translator = Translator(
+        #     engine="tencent",
+        #     cache_path=tencent_cache_dir,
+        #     api_id="YOUR_TENCENT_SECRET_ID",
+        #     api_key="YOUR_TENCENT_SECRET_KEY",
+        #     target_lang='zh',
+        #     source_lang='en'
+        # )
+        tencent_translator = Translator(engine="tencent", cache_path=tencent_cache_dir, target_lang='zh', source_lang='en')
+
+        text_to_translate_en = "Hello, world!"
+        print(f"Original (en): {text_to_translate_en}")
+
+        translated_zh = tencent_translator.translate(text_to_translate_en)
+        print(f"Translated (zh): {translated_zh}")
+
+        translated_zh_cached = tencent_translator.translate(text_to_translate_en)
+        print(f"Translated (zh) from cache: {translated_zh_cached}")
+        
+        assert translated_zh == translated_zh_cached
+
+        # tencent_translator.clear_cache()
+        tencent_translator.close_cache()
+        print("Tencent Translator test finished.\n")
+
+    except ValueError as ve:
+        print(f"Skipping Tencent test: {ve}")
+    except Exception as e:
+        print(f"Error during Tencent test: {e}")

toolkitx/text_utils.py

@@ -0,0 +1,160 @@
+import re
+
+
+# Define sentence terminators as a tuple for easy checking
+SENTENCE_TERMINATORS = ('.', '!', '?')
+
+def truncate_text_smart(text: str, limit: int = 100, mode: str = "char", suffix: str = "...",  tolerance: int = 10) -> str:
+    """
+    Smartly truncates text based on character or word limit, with tolerance.
+
+    :param text: The original string.
+    :param limit: The target truncation length (in characters or words).
+    :param mode: Truncation mode: 'char' for character-based, 'word' for word-based.
+    :param suffix: The suffix to append after truncation.
+    :param tolerance: The allowed deviation from the limit for smart truncation.
+    :return: The truncated string.
+    :raises ValueError: If the mode is not 'char' or 'word'.
+    """
+    if not isinstance(text, str):
+        # Or handle non-string input differently, e.g., convert to string or raise TypeError
+        return str(text) # Basic handling for non-string input
+
+    if mode == 'char':
+        # If text is already within or at the limit, no truncation needed.
+        if len(text) <= limit:
+            return text
+
+        # If the limit is too small to even hold the suffix,
+        # return the suffix truncated to the limit.
+        if limit <= len(suffix):
+            return suffix[:limit] if limit > 0 else ""
+
+        # Ideal length of the text part before the suffix.
+        ideal_text_part_len = limit - len(suffix)
+        # Maximum length of the text part we are willing to consider (within tolerance).
+        max_potential_text_len = min(len(text), limit + tolerance - len(suffix))
+        # Minimum length for the text part for smart truncation (within tolerance).
+        min_potential_text_len = max(0, limit - tolerance - len(suffix))
+        # Ensure min_potential_text_len is not greater than max_potential_text_len.
+        min_potential_text_len = min(min_potential_text_len, max_potential_text_len)
+
+        # The chunk of text to search for smart cut points.
+        # We search up to max_potential_text_len.
+        candidate_chunk_for_search = text[:max_potential_text_len]
+
+        # Attempt 1: Find a sentence boundary.
+        # Search backwards for the last sentence terminator in the candidate_chunk.
+        # The cut should result in a text part whose length is between min_potential_text_len and max_potential_text_len.
+        best_sentence_cut_len = -1
+        for i in range(len(candidate_chunk_for_search) - 1, -1, -1):
+            char_at_i = candidate_chunk_for_search[i]
+            # Check if it's a sentence terminator and the resulting part is long enough.
+            if char_at_i in SENTENCE_TERMINATORS:
+                # The length of the text part would be i + 1.
+                current_cut_len = i + 1
+                if current_cut_len >= min_potential_text_len:
+                    # Check if it's a proper sentence end (e.g., followed by space or end of chunk)
+                    # This check is simplified; more robust NLP might be needed for edge cases like "U.S.A."
+                    is_actual_sentence_end = (i + 1 == len(candidate_chunk_for_search)) or \
+                                             (i + 1 < len(candidate_chunk_for_search) and candidate_chunk_for_search[i+1] == ' ')
+                    if is_actual_sentence_end:
+                        best_sentence_cut_len = current_cut_len
+                        break # Found the latest possible sentence cut within tolerance.
+        
+        if best_sentence_cut_len != -1:
+            # .rstrip() to handle cases like "Sentence.  " before adding suffix.
+            return text[:best_sentence_cut_len].rstrip() + suffix
+
+        # Attempt 2: Find a word boundary.
+        # Search backwards for the last space in the candidate_chunk.
+        # The cut should result in a text part whose length is between min_potential_text_len and max_potential_text_len.
+        best_word_cut_len = -1
+        for i in range(len(candidate_chunk_for_search) - 1, -1, -1):
+            char_at_i = candidate_chunk_for_search[i]
+            if char_at_i == ' ':
+                # The length of the text part would be i (cutting before the space).
+                current_cut_len = i
+                if current_cut_len >= min_potential_text_len:
+                    best_word_cut_len = current_cut_len
+                    break # Found the latest possible word cut within tolerance.
+
+        if best_word_cut_len != -1:
+            # .rstrip() just in case, though text[:best_word_cut_len] should not have trailing spaces.
+            return text[:best_word_cut_len].rstrip() + suffix
+            
+        # Fallback: Hard truncate to the ideal_text_part_len.
+        return text[:ideal_text_part_len] + suffix
+
+    elif mode == 'word':
+        words = text.split()
+
+        # If word count is already within or at the limit, no truncation needed.
+        if len(words) <= limit:
+            return text
+
+        best_word_count_for_cut = -1
+
+        # Attempt 1: Find a sentence boundary within word tolerance.
+        # Iterate from longest possible (limit + tolerance) down to shortest (limit - tolerance).
+        # Ensure k is at least 1.
+        start_k = min(len(words), limit + tolerance)
+        end_k = max(1, limit - tolerance)
+
+        for k in range(start_k, end_k - 1, -1):
+            if k == 0: continue # Should not happen with max(1, ...)
+            current_phrase_words = words[:k]
+            current_phrase_str = " ".join(current_phrase_words)
+            # Check if the formed phrase ends with a sentence terminator.
+            if current_phrase_str.rstrip().endswith(SENTENCE_TERMINATORS):
+                best_word_count_for_cut = k
+                break # Found the longest suitable sentence-ending phrase.
+        
+        if best_word_count_for_cut != -1:
+            final_words = words[:best_word_count_for_cut]
+            result_text = " ".join(final_words)
+            # Add suffix only if actual truncation happened relative to original word count.
+            if len(words) > len(final_words):
+                result_text += suffix
+            return result_text
+
+        # Fallback: Truncate to the 'limit' number of words.
+        # This also handles cases where no sentence boundary was found in tolerance.
+        final_words = words[:limit]
+        result_text = " ".join(final_words)
+        if len(words) > len(final_words): # Add suffix only if truncated
+            result_text += suffix
+        return result_text
+    else:
+        raise ValueError("mode must be 'char' or 'word'")
+ 
+ 
+
+def split_text_by_word_count(
+    text: str, max_words: int = 300, overlap: int = 0
+) -> list[str]:
+    """
+    Split a long text into overlapping chunks (trunks), each with at most `max_words` words,
+    and `overlap` words overlapping between consecutive trunks.
+
+    Args:
+        text (str): The input text.
+        max_words (int): Maximum number of words per chunk.
+        overlap (int): Number of overlapping words between adjacent chunks.
+
+    Returns:
+        List[str]: A list of text chunks.
+    """
+    assert 0 <= overlap < max_words, "Overlap must be >= 0 and less than max_words"
+
+    words = text.split()
+    trunks = []
+    step = max_words - overlap
+
+    for i in range(0, len(words), step):
+        trunk = " ".join(words[i : i + max_words])
+        trunks.append(trunk)
+        if i + max_words >= len(words):
+            break
+
+    return trunks

toolkitx-0.0.1.dist-info/METADATA

@@ -0,0 +1,66 @@
+Metadata-Version: 2.4
+Name: toolkitx
+Version: 0.0.1
+Summary: A personal Python toolkit for common tasks
+Project-URL: Homepage, https://github.com/ider-zh/toolkitx
+Project-URL: Bug Tracker, https://github.com/ider-zh/toolkitx/issues
+License-File: LICENSE
+Requires-Python: >=3.12
+Requires-Dist: diskcache>=5.6.3
+Requires-Dist: py-transgpt>=2.0
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Requires-Dist: pytest-dotenv; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# toolkitx
+
+A personal Python toolkit for common tasks. This package provides various utility functions to simplify common development workflows.
+
+## Features
+
+*   **Text Utilities** (`toolkitx.text_utils`):
+    *   `truncate_text_smart`: Smartly truncates text by characters or words, with options for suffix and tolerance, attempting to preserve sentence or word boundaries.
+    *   `split_text_by_word_count`: Splits long text into overlapping chunks based on word count.
+*   **Command-Line Script**:
+    *   `hello`: A simple script accessible via `hello` command after installation, prints a greeting message.
+*   **Experimental Translator** (`toolkitx.lab.translator`):
+    *   `Translator`: A class providing translation capabilities using Baidu or Tencent translation APIs, with disk-based caching for performance. (Requires API credentials)
+
+## Installation
+
+1.  Clone the repository:
+    ```bash
+    git clone https://github.com/ider-zh/toolkitx.git
+    cd toolkitx
+    ```
+2.  Install the package. For development, you can install it in editable mode with development dependencies:
+    ```bash
+    pip install -e ".[dev]"
+    ```
+    For regular installation:
+    ```bash
+    pip install .
+    ```
+
+## Usage
+
+### Text Utilities
+
+```python
+from toolkitx import truncate_text_smart, split_text_by_word_count
+
+# Smart Truncation
+text = "This is a very long sentence that needs to be truncated."
+truncated_char = truncate_text_smart(text, limit=20, mode="char", suffix="...")
+print(f"Char truncated: {truncated_char}")
+
+truncated_word = truncate_text_smart(text, limit=5, mode="word", suffix="...")
+print(f"Word truncated: {truncated_word}")
+
+# Split Text
+long_text = "This is a long piece of text that we want to split into several smaller chunks with some overlap between them for context."
+chunks = split_text_by_word_count(long_text, max_words=10, overlap=2)
+for i, chunk in enumerate(chunks):
+    print(f"Chunk {i+1}: {chunk}")
+```

toolkitx-0.0.1.dist-info/RECORD

@@ -0,0 +1,10 @@
+toolkitx/__init__.py,sha256=a06W3QTZABS6VQb6SiVzkBe35OzzrYMFrCVXXBLHOcM,459
+toolkitx/hello.py,sha256=_2zorlP0W9gLGuDUPO7MSx5Hd6szTJnWPE014vR6Wfw,47
+toolkitx/text_utils.py,sha256=MVK_Q8DGEUlOo7vz4cUOU4FKFsAoV-1pvgi-DNs-VCw,7393
+toolkitx/lab/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+toolkitx/lab/translator.py,sha256=L9yyAz6SbwbfggznaSF_7P2NetL0RqLBmlVqn5inxIU,9547
+toolkitx-0.0.1.dist-info/METADATA,sha256=K8onr_V6_D31-S-Nbf-JMUb61aBstBLPthW-6U7auE0,2430
+toolkitx-0.0.1.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+toolkitx-0.0.1.dist-info/entry_points.txt,sha256=qzg8ZI4SSi3YrHbBu7OOa6j8hRiTwKfRv2tXrDEIdSg,47
+toolkitx-0.0.1.dist-info/licenses/LICENSE,sha256=awOCsWJ58m_2kBQwBUGWejVqZm6wuRtCL2hi9rfa0X4,1211
+toolkitx-0.0.1.dist-info/RECORD,,

toolkitx-0.0.1.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

toolkitx-0.0.1.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+hello = toolkitx.hello:hello

toolkitx-0.0.1.dist-info/licenses/LICENSE

@@ -0,0 +1,24 @@
+This is free and unencumbered software released into the public domain.
+
+Anyone is free to copy, modify, publish, use, compile, sell, or
+distribute this software, either in source code form or as a compiled
+binary, for any purpose, commercial or non-commercial, and by any
+means.
+
+In jurisdictions that recognize copyright laws, the author or authors
+of this software dedicate any and all copyright interest in the
+software to the public domain. We make this dedication for the benefit
+of the public at large and to the detriment of our heirs and
+successors. We intend this dedication to be an overt act of
+relinquishment in perpetuity of all present and future rights to this
+software under copyright law.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
+OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+OTHER DEALINGS IN THE SOFTWARE.
+
+For more information, please refer to <https://unlicense.org>