opentrans 0.1.0__py3-none-any.whl

opentrans/__init__.py

@@ -0,0 +1,6 @@
+__version__ = "v0.1.0"
+
+from .cache_manager import CacheManager
+from .hasher import get_file_hash
+
+__all__ = ['get_file_hash']

opentrans/cache_manager.py

@@ -0,0 +1,68 @@
+import json
+import os
+from pathlib import Path
+from .hasher import get_file_hash
+from typing import Union
+from .settings import settings
+
+class CacheManager:
+  """
+  Change Cache
+  """
+  def __init__(self, cache_file: Path):
+    self.cache_file = cache_file
+    self.cache = self._load_cache()
+  
+  def __del__(self):
+    self.save() 
+
+  def _load_cache(self) -> dict:
+    """
+    Docstring for _load_cache
+    
+    :return: file_path and hash in dict format from cache file
+    :rtype: dict
+    """ 
+    if self.cache_file == None:
+      return {}
+
+    if not self.cache_file.exists():
+      self.cache_file.parent.mkdir(parents=True, exist_ok=True)
+      with open(self.cache_file, 'w') as f:
+        json.dump({}, f, indent=2)
+    
+    try:
+      with open(self.cache_file, "r") as f:
+        return json.load(f)
+    except (json.JSONDecodeError, IOError):
+      return {}
+    return {}
+  
+  def file_changed(self, file_path: Union[Path, str]) -> bool:
+    """
+    Check if the hash changed
+
+    Args:
+        file_path (Path): file to check
+
+    Returns:
+        bool: True if changed else False
+    """
+    return self.cache.get(str(file_path)) != get_file_hash(file_path, settings.hash_algo)
+  
+  def __contains__(self, file_path: Union[Path, str]):
+    return str(file_path) in self.cache
+   
+  def update(self, file_path: Path):
+    """
+    Update the file hash in cache. Create the file if not exist.
+
+    Args:
+        file_path (Path): File to update
+    """
+    self.cache[str(file_path)] = get_file_hash(file_path, settings.hash_algo)
+  
+  def save(self):
+    with open(self.cache_file, "w") as f:
+      json.dump(self.cache, f, indent=2)
+        

opentrans/hasher.py

@@ -0,0 +1,27 @@
+import hashlib
+from pathlib import Path
+from typing import Union
+
+
+def get_file_hash(file_path: Union[str, Path], algorithm="sha256") -> str:
+  """
+  Generate a hash(default sha256) for a file's content.
+
+  Args:
+      file_path (str | Path): File to hash.
+
+  Returns:
+      str: hash of the file
+  """
+  try:
+    with open(file_path, "rb") as f:
+      if hasattr(hashlib, "file_digest"):
+        digest = hashlib.file_digest(f, algorithm)
+      else:
+        digest = hashlib.new(algorithm)
+        for chunk in iter(lambda: f.read(8192), b""):
+            digest.update(chunk)
+
+      return digest.hexdigest()
+  except FileNotFoundError:
+      raise FileNotFoundError(f"Error: File not found at {file_path}")

opentrans/main.py

@@ -0,0 +1,98 @@
+import click
+from pathlib import Path
+from concurrent.futures import ThreadPoolExecutor
+from tqdm import tqdm
+
+from .settings import settings
+from .translator import Translator
+
+
+@click.command()
+@click.argument(
+    'input_dir',
+    type=click.Path(exists=True, path_type=Path)
+)
+@click.argument(
+    'output_dir',
+    type=click.Path(path_type=Path)
+)
+@click.argument(
+    'config',
+    type=click.Path(path_type=Path)
+)
+@click.option('--model', '-m', default='translategemma:4b', help='Ollama model name.')
+@click.option('--lang', '-l', default='Chinese', help='Target translation language.')
+@click.option(
+    '--input_dir', '-i',
+    type=click.Path(exists=True, path_type=Path),
+    help='Directory containing the documents to translate.'
+)
+@click.option(
+    '--output_dir', '-o',
+    type=click.Path(path_type=Path),
+    help='Directory where translated files will be saved.'
+)
+@click.option(
+    '--config', '-c',
+    type=click.Path(path_type=Path),
+    help='Custom path to a config.yaml settings file.'
+)
+
+def main(input_dir, output_dir, config, model, lang):
+    """
+    OpenTrans: Batch translate documentation using local Ollama LLMs.
+
+    A specialized tool for translating Markdown and LaTeX files while preserving 
+    input directory structures and protecting technical syntax (code, math, and links).
+
+    Key Features:
+    - Local-First: Powered by Ollama for private, cost-free translation.
+    - Syntax Shield: Automatically protects backticks, LaTeX, and URLs from translation.
+    - Parallel Processing: High-speed batch handling for large documentation sets.
+    - Structure Preservation: Mirror your source directory exactly in the output.
+    """
+    if config:
+      settings.model_config['yaml_file'] = config
+
+    if input_dir:
+      settings.input_dir = input_dir.resolve()
+
+    if output_dir:
+      settings.output_dir = output_dir.resolve()
+    
+    if model:
+      settings.ollama_model = model
+    
+    if lang:
+      settings.target_lang = lang
+
+    if not settings.input_dir.exists():
+      click.echo(
+        f"Error: Input directory {settings.input_dir} does not exist.", err=True)
+      return
+
+    all_files = [f for f in settings.input_dir.rglob('*') if f.is_file()]
+
+    click.echo(f"Target Language: {settings.target_lang}")
+    click.echo(f"Input:  {settings.input_dir}")
+    click.echo(f"Output: {settings.output_dir}")
+    click.echo(f"Using Model: {settings.ollama_model}")
+
+    translator = Translator(output_dir / settings.cache_filename)
+    with ThreadPoolExecutor(max_workers=settings.max_parallel_files) as executor:
+      list(tqdm(
+        executor.map(
+          lambda f: translator.process_file(
+              f, settings.input_dir, settings.output_dir),
+          all_files
+        ),
+        total=len(all_files),
+        desc="Translating Files",
+        unit="file"
+      ))
+
+    click.echo("\nComplete.")
+
+
+if __name__ == '__main__':
+    main()

opentrans/settings.py

@@ -0,0 +1,64 @@
+import os
+from pathlib import Path
+from typing import Set, Tuple, Type
+
+from pydantic import computed_field, field_validator, Field
+from pydantic_settings import (
+    BaseSettings,
+    PydanticBaseSettingsSource,
+    SettingsConfigDict,
+    YamlConfigSettingsSource,
+)
+
+class Settings(BaseSettings):
+    target_lang: str = "Chinese"
+    ollama_model: str = "translategemma:4b"
+    temperature: float = Field(default=1.0, ge=0.0, le=2.0)
+    max_parallel_files: int = Field(default=2, gt=0)
+    
+    input_dir: Path = Path("./docs").resolve()
+    output_dir: Path = Path("./translated").resolve()
+    hash_algo: str = "sha1"
+    translate_file_types: Set[str] = {".md", ".mdx", ".txt"}
+
+    model_config = SettingsConfigDict(
+        yaml_file="config.yaml",
+        extra="ignore",
+    )
+
+    @computed_field
+    @property
+    def cache_filename(self) -> str:
+        return f".{self.target_lang.lower()}_cache.json"
+
+    @property
+    def cache_path(self) -> Path:
+        return self.output_dir / self.cache_filename
+
+    @classmethod
+    def check_input_directory(cls, v: Path) -> Path:
+        if not v.exists():
+            print(f"Warning: Input directory {v} does not exist.")
+        return v
+
+    @classmethod
+    def settings_customise_sources(
+        cls,
+        settings_cls: Type[BaseSettings],
+        init_settings: PydanticBaseSettingsSource,
+        env_settings: PydanticBaseSettingsSource,
+        dotenv_settings: PydanticBaseSettingsSource,
+        file_secret_settings: PydanticBaseSettingsSource,
+    ) -> Tuple[PydanticBaseSettingsSource, ...]:
+        """Defines the priority: 1. Init, 2. Env Vars, 3. YAML File, 4. Defaults"""
+        return (
+            init_settings,
+            env_settings,
+            YamlConfigSettingsSource(settings_cls),
+        )
+
+try:
+    settings = Settings()
+except Exception as e:
+    print(f"Configuration Error: {e}")
+    raise

opentrans/translator.py

@@ -0,0 +1,191 @@
+import sys
+from tqdm import tqdm
+import ollama
+from pathlib import Path
+from shutil import copy2
+import threading
+from .settings import settings
+from .cache_manager import CacheManager
+import logging
+import re
+logger = logging.getLogger(__name__)
+
+
+class Translator:
+    def __init__(self, cache_path: Path):
+        self.cache = CacheManager(cache_path)
+        self.lock = threading.Lock()
+
+    def protect_syntax(self, text: str, protect_callback: callable):
+      """
+      Scans text for Markdown/Docusaurus syntax and replaces it with placeholders.
+
+      This function identifies technical and structural syntax—such as code blocks, 
+      LaTeX math, Docusaurus admonitions, and JSX components—and applies a 
+      protection callback to prevent them from being modified during translation. 
+      It ensures that technical syntax remains valid while allowing the 
+      surrounding prose to be localized.
+      
+      Args:
+        text (str): The raw Markdown or MDX content to be processed.
+        protect_callback (callable): A function or method that takes the 
+            matched string and returns a placeholder string (e.g., `[PROTECTED_0]`).
+
+      Returns:
+          str: The processed text with all identified technical syntax replaced 
+              by placeholders.
+      """
+      
+      # TODO: Replace this with tree-sitter or similar to redact code blocks depends on the file extension
+      # List of patterns to protect (Order matters! Larger blocks first)
+      patterns = [
+        r'(?s:^---[\s\S]+?---)',                                          # Frontmatter
+        r'(?s:```[\s\S]*?```)',                                           # Code Blocks
+        r'(?s:<CodeBlock[^>]*>[\s\S]*?</CodeBlock>)',                     # Docusaurus CodeBlock
+        r'(?m:^:::[a-zA-Z]+(?:\[.*?\])?\s*$)',                            # Admonition Start
+        r'(?m:^:::\s*$)',                                                 # Admonition End
+        r'(?s:<[a-zA-Z][\w:.-]*(?:\s+[a-zA-Z0-9:-]+=(?:"[^"]*"|\'[^\']*\'|\{[^}]*\}|[^ >]+))*\s*/?>)', # Opening Tags (All HTML/JSX)
+        r'(?m:</[a-zA-Z][\w:.-]*\s*>)',                                   # Closing Tags (All HTML/JSX)
+        r'(?s:\$\$[\s\S]*?\$\$)',                                         # Display Math
+        r'(?<!\$)\$[^\$\n]+\$(?!\$)',                                     # Inline Math
+        r'`[^`\n]+`',                                                     # Inline Code
+        r'(?<=\]\()([^\)\s]+)(?=\))',                                     # Link URLs
+        r'(?m:^import\s+.*?;$)'                                           # MDX Imports
+      ]
+      combined_pattern = '|'.join(f'(?:{p})' for p in patterns)
+      return re.sub(combined_pattern, lambda m: protect_callback(m.group(0)), text)
+  
+    def translate_text(self, text: str, target_lang: str = settings.target_lang, model: str = settings.ollama_model) -> str:
+        """
+        Sends text to the Ollama API for translation using a specified model,
+        protecting technical syntax with temporary placeholders.
+
+        Args:
+          text (str): The raw content of the file to be translated.
+          target_lang (str): The language to translate the text into (e.g., 'Chinese').
+          model (str): The name of the Ollama model to use for translation.
+
+        Returns:
+          str: The translated text content with original technical syntax preserved.
+        """
+        placeholders = {}
+
+        def protect(text_content):
+            placeholder = f"[[DOC_REF_{len(placeholders)}]]"
+            placeholders[placeholder] = text_content # No need for .group(0)
+            return placeholder
+
+        protected_text = self.protect_syntax(text, protect)
+        
+        system_prompt = (
+            f"You are an expert technical translator. Translate the document into {target_lang}.\n"
+            "CRITICAL: Do not modify or translate tokens like [[DOC_REF_N]].\n"
+            "Preserve all structural symbols. Output ONLY the translated text."
+        )
+
+        user_prompt = f"TEXT TO TRANSLATE:\n\n{protected_text}"
+
+        try:
+            ollama.chat(model)
+        except ollama.ResponseError as e:
+            if e.status_code == 404:
+                self.pull_model_with_progress(model)
+
+        messages = [
+            {'role': 'system', 'content': system_prompt},
+            {'role': 'user', 'content': user_prompt}
+        ]
+
+        try:
+            response = ollama.chat(
+                model=model,
+                messages=messages,
+                options={"temperature": settings.temperature}
+            )
+            translated = response['message']['content'].strip()
+
+            for placeholder, original_value in placeholders.items():
+                translated = translated.replace(placeholder, original_value)
+
+            return translated
+
+        except Exception as e:
+            logger.error("Translation Failed.")
+            logger.info(e)
+            return text
+
+    def process_file(self, file_path: Path, input_root: Path, output_root: Path):
+        """
+        Handles the logic for a single file: translates it if it matches allowed 
+        extensions, otherwise copies it directly to the output directory.
+
+        Args:
+          file_path (Path): The path to the source file.
+          input_root (Path): The root directory of the source files for relative path calculation.
+          output_root (Path): The root directory where translated/copied files will be saved.
+        """
+        new_file_path = output_root / file_path.relative_to(input_root)
+
+        if not self.cache.file_changed(file_path):
+            return
+
+        new_file_path.parent.mkdir(parents=True, exist_ok=True)
+
+        if file_path.suffix in settings.translate_file_types:
+            content = file_path.read_text(encoding='utf-8')
+            translated = self.translate_text(
+                content, settings.target_lang, settings.ollama_model)
+            new_file_path.write_text(translated, encoding='utf-8')
+        else:
+            copy2(file_path, new_file_path)
+
+        self.cache.update(file_path)
+
+    def pull_model_with_progress(self, model_name: str):
+        """
+        Downloads a model from the Ollama registry while displaying a real-time 
+        progress bar using tqdm for each layer.
+
+        Args:
+          model_name (str): The name of the model to pull (e.g., 'translategemma:4b').
+
+        Raises:
+          SystemExit: Exits the program with code 1 if the download fails.
+        """
+        print(f"Attempting to pull model: {model_name}")
+        logger.info(f"Attempting to pull model: {model_name}")
+        current_digest = ''
+        progress_bars = {}
+
+        try:
+            for progress in ollama.pull(model_name, stream=True):
+                digest = progress.get('digest', '')
+
+                if digest != current_digest and current_digest in progress_bars:
+                    progress_bars[current_digest].close()
+
+                if not digest:
+                    print(progress.get('status'))
+                    continue
+
+                if digest not in progress_bars and (total := progress.get('total')):
+                    progress_bars[digest] = tqdm(
+                        total=total,
+                        desc=f'Downloading {digest[7:19]}',
+                        unit='B',
+                        unit_scale=True
+                    )
+
+                if completed := progress.get('completed'):
+                    progress_bars[digest].update(
+                        completed - progress_bars[digest].n)
+
+                current_digest = digest
+
+            for bar in progress_bars.values():
+                bar.close()
+            print(f"Model {model_name} pull complete.")
+
+        except Exception as e:
+            print(f"An error occurred while pulling model: {e}")
+            sys.exit(1)

opentrans-0.1.0.dist-info/METADATA

@@ -0,0 +1,182 @@
+Metadata-Version: 2.4
+Name: opentrans
+Version: 0.1.0
+Summary: A directory-preserving AI translator using Ollama.
+Project-URL: Homepage, https://github.com/RainStorm108/opentrans
+Project-URL: Documentation, https://rainstorm108.github.io/opentrans/
+Project-URL: Issues, https://github.com/RainStorm108/opentrans/issues
+Author: Hydrangea
+License: MIT
+License-File: LICENSE.md
+Keywords: ai,automation,llm,ollama,translation
+Classifier: Development Status :: 4 - Beta
+Classifier: Intended Audience :: Developers
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.9
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Topic :: Text Processing :: Linguistic
+Requires-Python: >=3.9
+Requires-Dist: click
+Requires-Dist: ollama>=0.1.0
+Requires-Dist: pydantic
+Requires-Dist: pydantic-settings
+Requires-Dist: pyyaml
+Requires-Dist: tqdm>=4.66.0
+Description-Content-Type: text/markdown
+
+# OpenTrans
+
+**OpenTrans** is a privacy-first, local-LLM batch translation utility using Ollama. It is designed to mirror a source directory into a target language while preserving the exact file hierarchy.
+
+## Key Features
+
+* **Local-First (Ollama):** Private, cost-free translation using models like Gemma, Llama 3, or DeepSeek.
+* **Syntax Shielding:** Automatically protects code blocks (```), inline code (`), LaTeX math ($), and Markdown links from being corrupted by the LLM.
+* **Directory Mirroring:** Recursively replicates your source folder structure in the output destination.
+* **Parallel Processing:** Uses `ThreadPoolExecutor` for high-speed batch handling of large file sets.
+* **Smart Caching:** Uses hashing to track file changes. Only files that have been modified since the last run are sent to the LLM, saving significant time and compute resources.
+* **Resilient File Handling:** Automatically pulls required models from Ollama with a real-time progress bar if they are missing.
+
+## Quick Usage
+
+Simply provide the input directory and the target output directory.
+
+```bash
+opentrans ./docs ./translated_docs --config ./config.yaml
+
+```
+
+**Example Output:**
+
+```text
+Target Language: Chinese
+Input:  /home/user/Projects/OpenTrans/docs
+Output: /home/user/Projects/OpenTrans/translated_docs
+Using Model: translategemma:4b
+
+Translating Files: 100%|████████████████████████████████| 12/12 [00:45<00:00, 3.7s/file]
+
+Complete.
+```
+
+## Running Examples
+
+```bash
+# Translating Docusaurus
+opentrans ./Example/Docusaurus/docs ./Example/Docusaurus/i18n/zh-hans/docusaurus-plugin-content-blog/current ./config.yaml    
+```
+
+## Installation
+
+### Setup 
+
+OpenTrans requires Ollama to be installed and running on your local machine.
+
+1. Install Ollama: Follow instructions at [Ollama](https://ollama.com/download)
+
+2. Install [uv](https://docs.astral.sh/uv/getting-started/installation/)
+
+```shell
+curl -LsSf https://astral.sh/uv/install.sh | sh
+uv tool install .
+```
+
+3. Install OpenTrans
+
+```bash
+git clone https://github.com/rainstorm108/OpenTrans.git
+cd OpenTrans
+uv pip install .
+```
+
+4. run
+
+```bash
+opentrans ./docs ./translated_docs --config ./config.yaml
+```
+
+### For Developers
+
+1. Environment Setup
+
+```bash
+uv sync
+source .venv/bin/activte
+hatch shell
+uv pip install -e .
+```
+
+2. Running Tests
+
+```bash
+hatch test
+```
+
+## Folder Structure
+
+```text
+OpenTrans/
+├── src/
+│   └── opentrans/
+│       ├── __init__.py        
+│       ├── main.py            # CLI interface using Click
+│       ├── settings.py        # Translate settings
+│       ├── translator.py      # Translate logic
+│       ├── cache_manager.py   # Manages translation caching using content hashing to skip unchanged files.
+│       └── hasher.py          # Hash file for cache
+├── tests/
+│   ├── test_cache_manager.py
+│   ├── test_hasher.py
+│   └── test_translator.py
+├── config.yaml                # Global settings (model, language, etc.)
+├── pyproject.toml
+└── README.md 
+
+```
+
+## Workflow
+
+```mermaid
+graph TD
+    %% Setup Phase
+    Start((Start)) --> CheckRunning{Ollama Running?}
+    CheckRunning -- No --> Err[Error: Start Ollama]
+    CheckRunning -- Yes --> CheckModel{Model Exists?}
+    
+    CheckModel -- No --> Pull[Download Model with Progress Bar]
+    Pull --> Init
+    CheckModel -- Yes --> Init[Load Paths & Config]
+
+    %% The Loop
+    Init --> NextFile{Next File?}
+    NextFile -- No --> Exit([Exit])
+    NextFile -- Yes --> IsTranslatable{File Type Supported?}
+
+    %% Branching Logic
+    IsTranslatable -- No --> Mirror[Copy Original File]
+    IsTranslatable -- Yes --> Shield[Apply Syntax Placeholders]
+    
+    Shield --> Trans[Translate via Ollama API]
+    Trans --> Restore[Restore Protected Code/Math]
+    Restore --> Save[Create Dir & Save File]
+    Save --> NextFile
+
+```
+
+## Todo
+
+* [x] Click-based CLI interface
+* [x] Placeholder-based syntax protection (Code/LaTeX)
+* [x] Multi-threaded parallel processing
+* [x] Implement hash caching to skip unchanged files
+* [ ] Finish the Docusaurus translate script
+* [ ] User Tree-sitter to replace the code blocks before translation instead of regex
+* [ ] Add support for more file types
+* [ ] ...
+
+## License
+
+This project is licensed under the MIT License.

opentrans-0.1.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+opentrans/__init__.py,sha256=yxYZKd_7mI-ZBUbBxEDxHC2BlENTzhPafm6xX6luUAg,126
+opentrans/cache_manager.py,sha256=_1ENDkZe6HJWDVw0A29l7gPIYf3KKrsALm-Ikd9xnHc,1661
+opentrans/hasher.py,sha256=-XXNYI6Gq2w9vlsHq7jJCdaJgJ9rIchszg6urwA1Fao,711
+opentrans/main.py,sha256=-NYjsG2FubPpCOz0vrID3ypyoeCLfMP23-eMlegk_1U,2912
+opentrans/settings.py,sha256=GNSPG0YUNHJ5R2RqLzV_ayxNQGZSjMYszvh6yqQRfO0,1891
+opentrans/translator.py,sha256=w2GGR_FBNhRub06vP9_IShqBrAZTB-Ej1wOCbicPouI,7883
+opentrans-0.1.0.dist-info/METADATA,sha256=H__fK87tJQdtGGKSXPTYvJxm-y7bI9PbfVnKpns00zA,5461
+opentrans-0.1.0.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87
+opentrans-0.1.0.dist-info/entry_points.txt,sha256=u_CT_SOehQGyiqNMN2AiFDGMLzZ4g-o_ayIH6WWJcgA,50
+opentrans-0.1.0.dist-info/licenses/LICENSE.md,sha256=Gmw7jhwvehQCdjZXF9KkTTKdCnlci3uEUvGsVh_fl4Q,1069
+opentrans-0.1.0.dist-info/RECORD,,

opentrans-0.1.0.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: hatchling 1.27.0
+Root-Is-Purelib: true
+Tag: py3-none-any

opentrans-0.1.0.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[console_scripts]
+opentrans = opentrans.main:main

opentrans-0.1.0.dist-info/licenses/LICENSE.md

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 rainstorm108
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.