yaku 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. yaku-0.1.0/.gitignore +225 -0
  2. yaku-0.1.0/LICENSE +21 -0
  3. yaku-0.1.0/PKG-INFO +123 -0
  4. yaku-0.1.0/README.ja.md +96 -0
  5. yaku-0.1.0/README.md +96 -0
  6. yaku-0.1.0/pyproject.toml +72 -0
  7. yaku-0.1.0/src/yaku/__init__.py +1 -0
  8. yaku-0.1.0/src/yaku/__main__.py +5 -0
  9. yaku-0.1.0/src/yaku/cache.py +114 -0
  10. yaku-0.1.0/src/yaku/classify.py +95 -0
  11. yaku-0.1.0/src/yaku/cli.py +244 -0
  12. yaku-0.1.0/src/yaku/engines/__init__.py +5 -0
  13. yaku-0.1.0/src/yaku/engines/ct2.py +94 -0
  14. yaku-0.1.0/src/yaku/loader.py +144 -0
  15. yaku-0.1.0/src/yaku/masking.py +169 -0
  16. yaku-0.1.0/src/yaku/paths.py +79 -0
  17. yaku-0.1.0/src/yaku/pipeline.py +396 -0
  18. yaku-0.1.0/src/yaku/quality.py +57 -0
  19. yaku-0.1.0/src/yaku/rules/source/en.toml +27 -0
  20. yaku-0.1.0/src/yaku/rules/syntax/common.toml +171 -0
  21. yaku-0.1.0/src/yaku/rules/syntax/kubectl.toml +49 -0
  22. yaku-0.1.0/src/yaku/rules/syntax/man.toml +4 -0
  23. yaku-0.1.0/src/yaku/rules/syntax/terraform.toml +49 -0
  24. yaku-0.1.0/src/yaku/rules/target/ja.toml +31 -0
  25. yaku-0.1.0/src/yaku/setup.py +141 -0
  26. yaku-0.1.0/src/yaku/text.py +103 -0
  27. yaku-0.1.0/src/yaku/types.py +123 -0
  28. yaku-0.1.0/tests/conftest.py +24 -0
  29. yaku-0.1.0/tests/test_cache.py +59 -0
  30. yaku-0.1.0/tests/test_classify.py +62 -0
  31. yaku-0.1.0/tests/test_loader.py +89 -0
  32. yaku-0.1.0/tests/test_masking.py +221 -0
  33. yaku-0.1.0/tests/test_paths.py +36 -0
  34. yaku-0.1.0/tests/test_pipeline.py +318 -0
  35. yaku-0.1.0/tests/test_quality.py +82 -0
  36. yaku-0.1.0/tests/test_rules.py +341 -0
  37. yaku-0.1.0/tests/test_setup.py +98 -0
  38. yaku-0.1.0/tests/test_text.py +180 -0
yaku-0.1.0/.gitignore ADDED
@@ -0,0 +1,225 @@
1
+ # Byte-compiled / optimized / DLL files
2
+ __pycache__/
3
+ *.py[codz]
4
+ *$py.class
5
+
6
+ # C extensions
7
+ *.so
8
+
9
+ # Distribution / packaging
10
+ .Python
11
+ build/
12
+ develop-eggs/
13
+ dist/
14
+ downloads/
15
+ eggs/
16
+ .eggs/
17
+ lib/
18
+ lib64/
19
+ parts/
20
+ sdist/
21
+ var/
22
+ wheels/
23
+ share/python-wheels/
24
+ *.egg-info/
25
+ .installed.cfg
26
+ *.egg
27
+ MANIFEST
28
+
29
+ # PyInstaller
30
+ # Usually these files are written by a python script from a template
31
+ # before PyInstaller builds the exe, so as to inject date/other infos into it.
32
+ *.manifest
33
+ *.spec
34
+
35
+ # Installer logs
36
+ pip-log.txt
37
+ pip-delete-this-directory.txt
38
+
39
+ # Unit test / coverage reports
40
+ htmlcov/
41
+ .tox/
42
+ .nox/
43
+ .coverage
44
+ .coverage.*
45
+ .cache
46
+ nosetests.xml
47
+ coverage.xml
48
+ *.cover
49
+ *.py.cover
50
+ .hypothesis/
51
+ .pytest_cache/
52
+ cover/
53
+
54
+ # Translations
55
+ *.mo
56
+ *.pot
57
+
58
+ # Django stuff:
59
+ *.log
60
+ local_settings.py
61
+ db.sqlite3
62
+ db.sqlite3-journal
63
+
64
+ # Flask stuff:
65
+ instance/
66
+ .webassets-cache
67
+
68
+ # Scrapy stuff:
69
+ .scrapy
70
+
71
+ # Sphinx documentation
72
+ docs/_build/
73
+
74
+ # PyBuilder
75
+ .pybuilder/
76
+ /target/
77
+
78
+ # Jupyter Notebook
79
+ .ipynb_checkpoints
80
+
81
+ # IPython
82
+ profile_default/
83
+ ipython_config.py
84
+
85
+ # pyenv
86
+ # For a library or package, you might want to ignore these files since the code is
87
+ # intended to run in multiple environments; otherwise, check them in:
88
+ # .python-version
89
+
90
+ # pipenv
91
+ # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
92
+ # However, in case of collaboration, if having platform-specific dependencies or dependencies
93
+ # having no cross-platform support, pipenv may install dependencies that don't work, or not
94
+ # install all needed dependencies.
95
+ # Pipfile.lock
96
+
97
+ # UV
98
+ # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
99
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
100
+ # commonly ignored for libraries.
101
+ # uv.lock
102
+
103
+ # poetry
104
+ # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
105
+ # This is especially recommended for binary packages to ensure reproducibility, and is more
106
+ # commonly ignored for libraries.
107
+ # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
108
+ # poetry.lock
109
+ # poetry.toml
110
+
111
+ # pdm
112
+ # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
113
+ # pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
114
+ # https://pdm-project.org/en/latest/usage/project/#working-with-version-control
115
+ # pdm.lock
116
+ # pdm.toml
117
+ .pdm-python
118
+ .pdm-build/
119
+
120
+ # pixi
121
+ # Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
122
+ # pixi.lock
123
+ # Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
124
+ # in the .venv directory. It is recommended not to include this directory in version control.
125
+ .pixi
126
+
127
+ # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
128
+ __pypackages__/
129
+
130
+ # Celery stuff
131
+ celerybeat-schedule
132
+ celerybeat.pid
133
+
134
+ # Redis
135
+ *.rdb
136
+ *.aof
137
+ *.pid
138
+
139
+ # RabbitMQ
140
+ mnesia/
141
+ rabbitmq/
142
+ rabbitmq-data/
143
+
144
+ # ActiveMQ
145
+ activemq-data/
146
+
147
+ # SageMath parsed files
148
+ *.sage.py
149
+
150
+ # Environments
151
+ .env
152
+ .envrc
153
+ .venv
154
+ env/
155
+ venv/
156
+ ENV/
157
+ env.bak/
158
+ venv.bak/
159
+
160
+ # Spyder project settings
161
+ .spyderproject
162
+ .spyproject
163
+
164
+ # Rope project settings
165
+ .ropeproject
166
+
167
+ # mkdocs documentation
168
+ /site
169
+
170
+ # mypy
171
+ .mypy_cache/
172
+ .dmypy.json
173
+ dmypy.json
174
+
175
+ # Pyre type checker
176
+ .pyre/
177
+
178
+ # pytype static type analyzer
179
+ .pytype/
180
+
181
+ # Cython debug symbols
182
+ cython_debug/
183
+
184
+ # PyCharm
185
+ # JetBrains specific template is maintained in a separate JetBrains.gitignore that can
186
+ # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
187
+ # and can be added to the global gitignore or merged into this file. For a more nuclear
188
+ # option (not recommended) you can uncomment the following to ignore the entire idea folder.
189
+ # .idea/
190
+
191
+ # Abstra
192
+ # Abstra is an AI-powered process automation framework.
193
+ # Ignore directories containing user credentials, local state, and settings.
194
+ # Learn more at https://abstra.io/docs
195
+ .abstra/
196
+
197
+ # Visual Studio Code
198
+ # Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
199
+ # that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
200
+ # and can be added to the global gitignore or merged into this file. However, if you prefer,
201
+ # you could uncomment the following to ignore the entire vscode folder
202
+ # .vscode/
203
+ # Temporary file for partial code execution
204
+ tempCodeRunnerFile.py
205
+
206
+ # Ruff stuff:
207
+ .ruff_cache/
208
+
209
+ # PyPI configuration file
210
+ .pypirc
211
+
212
+ # Marimo
213
+ marimo/_static/
214
+ marimo/_lsp/
215
+ __marimo__/
216
+
217
+ # Streamlit
218
+ .streamlit/secrets.toml
219
+
220
+ .memo
221
+ .tours
222
+ .convert/
223
+ .venv/
224
+ models/
225
+ *.sqlite3
yaku-0.1.0/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Shuichi Ohsawa
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
yaku-0.1.0/PKG-INFO ADDED
@@ -0,0 +1,123 @@
1
+ Metadata-Version: 2.5
2
+ Name: yaku
3
+ Version: 0.1.0
4
+ Summary: Offline translator for command output that keeps identifiers intact
5
+ Project-URL: Homepage, https://github.com/ohsawa0515/yaku
6
+ Project-URL: Repository, https://github.com/ohsawa0515/yaku
7
+ Project-URL: Issues, https://github.com/ohsawa0515/yaku/issues
8
+ Author: Shuichi Ohsawa
9
+ License-Expression: MIT
10
+ License-File: LICENSE
11
+ Keywords: cli,nmt,offline,translation
12
+ Classifier: Development Status :: 3 - Alpha
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Natural Language :: Japanese
16
+ Classifier: Operating System :: MacOS
17
+ Classifier: Operating System :: POSIX :: Linux
18
+ Classifier: Programming Language :: Python :: 3
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Text Processing :: Linguistic
22
+ Classifier: Topic :: Utilities
23
+ Requires-Python: >=3.12
24
+ Requires-Dist: ctranslate2>=4.0
25
+ Requires-Dist: sentencepiece>=0.2
26
+ Description-Content-Type: text/markdown
27
+
28
+ # yaku
29
+
30
+ English | [日本語](README.ja.md)
31
+
32
+ Fully offline translator for command output. Only the prose gets translated; everything else is left verbatim.
33
+
34
+ Identifiers, flags, paths and quoted strings survive unchanged, because they are moved out of the way before the model sees the text and put back afterwards. No network access, no API keys, no data leaving the machine.
35
+
36
+ ## Install
37
+
38
+ ```console
39
+ pip install yaku
40
+ yaku setup --lang ja
41
+ ```
42
+
43
+ `yaku setup` downloads [FuguMT](https://huggingface.co/staka/fugumt-en-ja) at a pinned revision and converts it to a 62 MB int8 model. The model is not bundled because it is CC BY-SA 4.0. The converter needs torch, which yaku does not depend on, so the conversion runs in a throwaway environment created with [uv](https://docs.astral.sh/uv/).
44
+
45
+ `yaku setup` requires uv. Install it and put it on `PATH` before running the command.
46
+
47
+ ## Use
48
+
49
+ ```console
50
+ terraform plan 2>&1 | yaku
51
+ man tar | col -bx | yaku
52
+ kubectl describe pod web | yaku
53
+ pbpaste | yaku
54
+ ```
55
+
56
+ Useful options:
57
+
58
+ | Option | Effect |
59
+ | --- | --- |
60
+ | `--lang ja` | target language, which also selects the model and thresholds |
61
+ | `--fallback both\|original\|mark` | what to emit for a line that fails the quality check |
62
+ | `--show-reason` | append why a line failed |
63
+ | `--stats` | write a summary to stderr |
64
+ | `--dry-run` | show classification and masking without loading a model |
65
+ | `--join` / `--no-join` | override the automatic wrapped-line detection |
66
+ | `--width N` | wrap the translated text |
67
+ | `--mask-off NAME,...` | disable mask rules |
68
+
69
+ ## How it works
70
+
71
+ ```
72
+ stdin
73
+ ↓ strip ANSI escapes
74
+ ↓ analysis pass decides whether lines are wrapped prose
75
+ ↓ classify pass through / command example / translatable
76
+ ↓ join reassemble wrapped sentences
77
+ ↓ mask identifiers become T1, T2, ...
78
+ ↓ split back into sentences
79
+ ↓ cache keyed on the normalised masked text
80
+ ↓ translate CTranslate2 + SentencePiece
81
+ ↓ check lost placeholders, length ratio, repetition
82
+ ↓ unmask identifiers restored
83
+ stdout
84
+ ```
85
+
86
+ Two parts carry most of the value.
87
+
88
+ **Masking** is not about withholding text. A sequence model that has never seen `aws_instance.web_server_01` in training will hallucinate around it; `T1 will be created` is ordinary English and translates cleanly. Masking also cuts the request by 5 to 10 percent in tokens, and a placeholder that fails to come back is the only reliable sign that a clause was dropped.
89
+
90
+ **The quality check** exists because dropped clauses cannot be prevented. Each line is verified afterwards, and anything suspect is shown with its original text rather than silently replaced.
91
+
92
+ ## Limits
93
+
94
+ - Translation quality is bounded by a 60M parameter model. It will not match a large language model.
95
+ - Dropped clauses cannot be eliminated, only detected. Roughly 5 to 9 percent of translated lines in manual pages are flagged, of which some are real drops and some are Japanese simply being more compact than the ratio check allows.
96
+ - Whether lines are joined is decided once for the whole input. Inside record-per-line output, a paragraph is still rejoined when its lines are filled to the wrap column, but a wrap that lands just before a capitalised word is left split.
97
+ - Japanese output sometimes loses a space around a restored identifier.
98
+
99
+ ## Adding a language
100
+
101
+ `--lang` switches the model and the measured constants together, so a new language needs a profile, not just a model.
102
+
103
+ 1. Pick a CTranslate2-compatible model (Marian and OPUS-MT convert cleanly).
104
+ 2. `uv run tools/check_placeholders.py` to choose a placeholder format.
105
+ 3. `uv run tools/eval_translate.py` to measure the length ratio bounds.
106
+ 4. Write `src/yaku/rules/target/<lang>.toml`.
107
+
108
+ ## Development
109
+
110
+ ```console
111
+ uv sync
112
+ export YAKU_MODELS_DIR=models # use the checkout's model, not the installed one
113
+ uv run pytest
114
+ uv run ruff check src tests tools && uv run ruff format src tests tools
115
+ uv run tools/measure_wrap.py captured-output.txt
116
+ uv run tools/eval_translate.py captured-output.txt
117
+ ```
118
+
119
+ Rules live in `src/yaku/rules/` as TOML. Users can add their own in `~/.config/yaku/rules/` or point `--rules-dir` at a directory; shipped rules are never edited in place.
120
+
121
+ ## Licence
122
+
123
+ MIT. The default model, [FuguMT](https://huggingface.co/staka/fugumt-en-ja), is CC BY-SA 4.0 and is downloaded separately.
@@ -0,0 +1,96 @@
1
+ # yaku
2
+
3
+ [English](README.md) | 日本語
4
+
5
+ コマンド出力を翻訳する、完全オフラインのフィルタです。翻訳するのは文章の部分だけで、それ以外は原文のまま残します。
6
+
7
+ 識別子、フラグ、パス、引用符で囲まれた文字列は変わりません。モデルに渡す前に退避し、翻訳後に元へ戻すためです。ネットワーク通信も API キーも不要で、データがマシンの外に出ることはありません。
8
+
9
+ ## インストール
10
+
11
+ ```console
12
+ pip install yaku
13
+ yaku setup --lang ja
14
+ ```
15
+
16
+ `yaku setup` は [FuguMT](https://huggingface.co/staka/fugumt-en-ja) を固定したリビジョンで取得し、62 MB の int8 モデルに変換します。モデルは CC BY-SA 4.0 のため、同梱していません。変換には torch が必要ですが、yaku の依存には含めていません。そのため変換は使い捨ての環境で行い、その作成に [uv](https://docs.astral.sh/uv/) を使います。
17
+
18
+ `yaku setup` の実行には uv が必要です。uv を `PATH` 上に入れてから実行してください。
19
+
20
+ ## 使い方
21
+
22
+ ```console
23
+ terraform plan 2>&1 | yaku
24
+ man tar | col -bx | yaku
25
+ kubectl describe pod web | yaku
26
+ pbpaste | yaku
27
+ ```
28
+
29
+ 主なオプションは次のとおりです。
30
+
31
+ | オプション | 内容 |
32
+ | --- | --- |
33
+ | `--lang ja` | 翻訳先の言語です。モデルと閾値も、これに合わせて切り替わります |
34
+ | `--fallback both\|original\|mark` | 品質検査に通らなかった行の出力方法です |
35
+ | `--show-reason` | 検査に通らなかった理由を付けます |
36
+ | `--stats` | 集計を stderr に出力します |
37
+ | `--dry-run` | モデルを読み込まず、行の分類とマスクの結果を表示します |
38
+ | `--join` / `--no-join` | 折り返し行の自動判定を上書きします |
39
+ | `--width N` | 翻訳結果を指定の幅で折り返します |
40
+ | `--mask-off NAME,...` | マスク規則を無効にします |
41
+
42
+ ## 仕組み
43
+
44
+ ```
45
+ stdin
46
+ ↓ ANSI エスケープの除去
47
+ ↓ 解析パス 折り返された文章かどうかを判定
48
+ ↓ 分類 そのまま出力 / コマンド例 / 翻訳対象
49
+ ↓ 結合 折り返された文を再構成
50
+ ↓ マスク 識別子を T1, T2, ... に置き換え
51
+ ↓ 分割 文単位に戻す
52
+ ↓ キャッシュ 正規化したマスク後の文をキーにする
53
+ ↓ 翻訳 CTranslate2 + SentencePiece
54
+ ↓ 検査 プレースホルダの消失、長さ比、反復
55
+ ↓ 復元 識別子を元に戻す
56
+ stdout
57
+ ```
58
+
59
+ 価値の大部分は次の 2 つにあります。
60
+
61
+ マスクは、テキストをモデルに見せないためのものではありません。`aws_instance.web_server_01` のような文字列は、系列変換モデルが学習時に見ていない形なので、その周辺で誤った出力を生みます。`T1 will be created` は通常の英文なので、きれいに訳せます。マスクによってモデルへの入力はトークン数で 5 から 10 パーセント減り、プレースホルダが戻ってこないことが、節が訳抜けしたことを確実に示す唯一の手がかりになります。
62
+
63
+ 品質検査があるのは、訳抜けを防げないからです。各行を翻訳後に検査し、疑わしいものは通知なく置き換えず、原文を併記して表示します。
64
+
65
+ ## 制約
66
+
67
+ - 訳質は 60M パラメータのモデルの範囲にとどまり、大規模言語モデルには及びません。
68
+ - 訳抜けはなくせず、検出できるだけです。man ページでは翻訳した行の 5 から 9 パーセントほどが検査に引っかかります。その中には、本当に訳が抜けたものと、日本語が長さ比の想定より短くなっただけのものがあります。
69
+ - 行を結合するかどうかは、入力全体に対して 1 回だけ判定します。1 行 1 記録の出力の中でも、折り返し幅まで埋まった行からなる段落は再結合しますが、折り返し位置の直後が大文字で始まる語の場合は分かれたままになります。
70
+ - 日本語の出力で、復元した識別子の前後の空白が欠けることがあります。
71
+
72
+ ## 言語の追加
73
+
74
+ `--lang` はモデルと、計測した定数をまとめて切り替えます。新しい言語には、モデルだけでなくプロファイルが必要です。
75
+
76
+ 1. CTranslate2 に変換できるモデルを選びます(Marian と OPUS-MT は問題なく変換できます)。
77
+ 2. `uv run tools/check_placeholders.py` でプレースホルダの形式を選びます。
78
+ 3. `uv run tools/eval_translate.py` で長さ比の範囲を計測します。
79
+ 4. `src/yaku/rules/target/<lang>.toml` を書きます。
80
+
81
+ ## 開発
82
+
83
+ ```console
84
+ uv sync
85
+ export YAKU_MODELS_DIR=models # インストール済みではなく、チェックアウト内のモデルを使う
86
+ uv run pytest
87
+ uv run ruff check src tests tools && uv run ruff format src tests tools
88
+ uv run tools/measure_wrap.py captured-output.txt
89
+ uv run tools/eval_translate.py captured-output.txt
90
+ ```
91
+
92
+ 規則は `src/yaku/rules/` に TOML で置いています。独自の規則は `~/.config/yaku/rules/` に追加するか、`--rules-dir` でディレクトリを指定します。同梱の規則を直接編集することはありません。
93
+
94
+ ## ライセンス
95
+
96
+ MIT です。標準のモデルである [FuguMT](https://huggingface.co/staka/fugumt-en-ja) は CC BY-SA 4.0 で、別途ダウンロードします。
yaku-0.1.0/README.md ADDED
@@ -0,0 +1,96 @@
1
+ # yaku
2
+
3
+ English | [日本語](README.ja.md)
4
+
5
+ Fully offline translator for command output. Only the prose gets translated; everything else is left verbatim.
6
+
7
+ Identifiers, flags, paths and quoted strings survive unchanged, because they are moved out of the way before the model sees the text and put back afterwards. No network access, no API keys, no data leaving the machine.
8
+
9
+ ## Install
10
+
11
+ ```console
12
+ pip install yaku
13
+ yaku setup --lang ja
14
+ ```
15
+
16
+ `yaku setup` downloads [FuguMT](https://huggingface.co/staka/fugumt-en-ja) at a pinned revision and converts it to a 62 MB int8 model. The model is not bundled because it is CC BY-SA 4.0. The converter needs torch, which yaku does not depend on, so the conversion runs in a throwaway environment created with [uv](https://docs.astral.sh/uv/).
17
+
18
+ `yaku setup` requires uv. Install it and put it on `PATH` before running the command.
19
+
20
+ ## Use
21
+
22
+ ```console
23
+ terraform plan 2>&1 | yaku
24
+ man tar | col -bx | yaku
25
+ kubectl describe pod web | yaku
26
+ pbpaste | yaku
27
+ ```
28
+
29
+ Useful options:
30
+
31
+ | Option | Effect |
32
+ | --- | --- |
33
+ | `--lang ja` | target language, which also selects the model and thresholds |
34
+ | `--fallback both\|original\|mark` | what to emit for a line that fails the quality check |
35
+ | `--show-reason` | append why a line failed |
36
+ | `--stats` | write a summary to stderr |
37
+ | `--dry-run` | show classification and masking without loading a model |
38
+ | `--join` / `--no-join` | override the automatic wrapped-line detection |
39
+ | `--width N` | wrap the translated text |
40
+ | `--mask-off NAME,...` | disable mask rules |
41
+
42
+ ## How it works
43
+
44
+ ```
45
+ stdin
46
+ ↓ strip ANSI escapes
47
+ ↓ analysis pass decides whether lines are wrapped prose
48
+ ↓ classify pass through / command example / translatable
49
+ ↓ join reassemble wrapped sentences
50
+ ↓ mask identifiers become T1, T2, ...
51
+ ↓ split back into sentences
52
+ ↓ cache keyed on the normalised masked text
53
+ ↓ translate CTranslate2 + SentencePiece
54
+ ↓ check lost placeholders, length ratio, repetition
55
+ ↓ unmask identifiers restored
56
+ stdout
57
+ ```
58
+
59
+ Two parts carry most of the value.
60
+
61
+ **Masking** is not about withholding text. A sequence model that has never seen `aws_instance.web_server_01` in training will hallucinate around it; `T1 will be created` is ordinary English and translates cleanly. Masking also cuts the request by 5 to 10 percent in tokens, and a placeholder that fails to come back is the only reliable sign that a clause was dropped.
62
+
63
+ **The quality check** exists because dropped clauses cannot be prevented. Each line is verified afterwards, and anything suspect is shown with its original text rather than silently replaced.
64
+
65
+ ## Limits
66
+
67
+ - Translation quality is bounded by a 60M parameter model. It will not match a large language model.
68
+ - Dropped clauses cannot be eliminated, only detected. Roughly 5 to 9 percent of translated lines in manual pages are flagged, of which some are real drops and some are Japanese simply being more compact than the ratio check allows.
69
+ - Whether lines are joined is decided once for the whole input. Inside record-per-line output, a paragraph is still rejoined when its lines are filled to the wrap column, but a wrap that lands just before a capitalised word is left split.
70
+ - Japanese output sometimes loses a space around a restored identifier.
71
+
72
+ ## Adding a language
73
+
74
+ `--lang` switches the model and the measured constants together, so a new language needs a profile, not just a model.
75
+
76
+ 1. Pick a CTranslate2-compatible model (Marian and OPUS-MT convert cleanly).
77
+ 2. `uv run tools/check_placeholders.py` to choose a placeholder format.
78
+ 3. `uv run tools/eval_translate.py` to measure the length ratio bounds.
79
+ 4. Write `src/yaku/rules/target/<lang>.toml`.
80
+
81
+ ## Development
82
+
83
+ ```console
84
+ uv sync
85
+ export YAKU_MODELS_DIR=models # use the checkout's model, not the installed one
86
+ uv run pytest
87
+ uv run ruff check src tests tools && uv run ruff format src tests tools
88
+ uv run tools/measure_wrap.py captured-output.txt
89
+ uv run tools/eval_translate.py captured-output.txt
90
+ ```
91
+
92
+ Rules live in `src/yaku/rules/` as TOML. Users can add their own in `~/.config/yaku/rules/` or point `--rules-dir` at a directory; shipped rules are never edited in place.
93
+
94
+ ## Licence
95
+
96
+ MIT. The default model, [FuguMT](https://huggingface.co/staka/fugumt-en-ja), is CC BY-SA 4.0 and is downloaded separately.
@@ -0,0 +1,72 @@
1
+ [build-system]
2
+ requires = ["hatchling"]
3
+ build-backend = "hatchling.build"
4
+
5
+ [project]
6
+ name = "yaku"
7
+ description = "Offline translator for command output that keeps identifiers intact"
8
+ readme = "README.md"
9
+ requires-python = ">=3.12"
10
+ license = "MIT"
11
+ license-files = ["LICENSE"]
12
+ authors = [{ name = "Shuichi Ohsawa" }]
13
+ keywords = ["translation", "cli", "offline", "nmt"]
14
+ classifiers = [
15
+ "Development Status :: 3 - Alpha",
16
+ "Environment :: Console",
17
+ "Intended Audience :: Developers",
18
+ "Natural Language :: Japanese",
19
+ "Operating System :: MacOS",
20
+ "Operating System :: POSIX :: Linux",
21
+ "Programming Language :: Python :: 3",
22
+ "Programming Language :: Python :: 3.12",
23
+ "Programming Language :: Python :: 3.13",
24
+ "Topic :: Text Processing :: Linguistic",
25
+ "Topic :: Utilities",
26
+ ]
27
+ # Rule files are TOML: tomllib is in the standard library, and its literal
28
+ # strings hold regular expressions without escaping.
29
+ dependencies = [
30
+ "ctranslate2>=4.0",
31
+ "sentencepiece>=0.2",
32
+ ]
33
+ dynamic = ["version"]
34
+
35
+ [project.urls]
36
+ Homepage = "https://github.com/ohsawa0515/yaku"
37
+ Repository = "https://github.com/ohsawa0515/yaku"
38
+ Issues = "https://github.com/ohsawa0515/yaku/issues"
39
+
40
+ [project.scripts]
41
+ yaku = "yaku.cli:run"
42
+
43
+ [dependency-groups]
44
+ dev = ["pytest>=8", "ruff>=0.8"]
45
+
46
+ [tool.hatch.version]
47
+ path = "src/yaku/__init__.py"
48
+
49
+ # An allowlist, so local scratch files ignored outside .gitignore stay out.
50
+ [tool.hatch.build.targets.sdist]
51
+ only-include = ["src", "tests", "LICENSE", "README.md", "README.ja.md", "pyproject.toml"]
52
+
53
+ [tool.hatch.build.targets.wheel]
54
+ packages = ["src/yaku"]
55
+
56
+ [tool.pytest.ini_options]
57
+ testpaths = ["tests"]
58
+
59
+ [tool.ruff]
60
+ line-length = 100
61
+ target-version = "py312"
62
+
63
+ [tool.ruff.lint]
64
+ select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
65
+ ignore = [
66
+ "RUF001", # ambiguous characters are intended: box drawing, fullwidth punctuation
67
+ "RUF003",
68
+ ]
69
+
70
+ [tool.ruff.lint.per-file-ignores]
71
+ # Test inputs are real command output, which is as long as it is.
72
+ "tests/*" = ["E501"]
@@ -0,0 +1 @@
1
+ __version__ = "0.1.0"
@@ -0,0 +1,5 @@
1
+ """Support for python -m yaku."""
2
+
3
+ from yaku.cli import run
4
+
5
+ run()