yaku 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- yaku-0.1.0/.gitignore +225 -0
- yaku-0.1.0/LICENSE +21 -0
- yaku-0.1.0/PKG-INFO +123 -0
- yaku-0.1.0/README.ja.md +96 -0
- yaku-0.1.0/README.md +96 -0
- yaku-0.1.0/pyproject.toml +72 -0
- yaku-0.1.0/src/yaku/__init__.py +1 -0
- yaku-0.1.0/src/yaku/__main__.py +5 -0
- yaku-0.1.0/src/yaku/cache.py +114 -0
- yaku-0.1.0/src/yaku/classify.py +95 -0
- yaku-0.1.0/src/yaku/cli.py +244 -0
- yaku-0.1.0/src/yaku/engines/__init__.py +5 -0
- yaku-0.1.0/src/yaku/engines/ct2.py +94 -0
- yaku-0.1.0/src/yaku/loader.py +144 -0
- yaku-0.1.0/src/yaku/masking.py +169 -0
- yaku-0.1.0/src/yaku/paths.py +79 -0
- yaku-0.1.0/src/yaku/pipeline.py +396 -0
- yaku-0.1.0/src/yaku/quality.py +57 -0
- yaku-0.1.0/src/yaku/rules/source/en.toml +27 -0
- yaku-0.1.0/src/yaku/rules/syntax/common.toml +171 -0
- yaku-0.1.0/src/yaku/rules/syntax/kubectl.toml +49 -0
- yaku-0.1.0/src/yaku/rules/syntax/man.toml +4 -0
- yaku-0.1.0/src/yaku/rules/syntax/terraform.toml +49 -0
- yaku-0.1.0/src/yaku/rules/target/ja.toml +31 -0
- yaku-0.1.0/src/yaku/setup.py +141 -0
- yaku-0.1.0/src/yaku/text.py +103 -0
- yaku-0.1.0/src/yaku/types.py +123 -0
- yaku-0.1.0/tests/conftest.py +24 -0
- yaku-0.1.0/tests/test_cache.py +59 -0
- yaku-0.1.0/tests/test_classify.py +62 -0
- yaku-0.1.0/tests/test_loader.py +89 -0
- yaku-0.1.0/tests/test_masking.py +221 -0
- yaku-0.1.0/tests/test_paths.py +36 -0
- yaku-0.1.0/tests/test_pipeline.py +318 -0
- yaku-0.1.0/tests/test_quality.py +82 -0
- yaku-0.1.0/tests/test_rules.py +341 -0
- yaku-0.1.0/tests/test_setup.py +98 -0
- yaku-0.1.0/tests/test_text.py +180 -0
yaku-0.1.0/.gitignore
ADDED
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
# Byte-compiled / optimized / DLL files
|
|
2
|
+
__pycache__/
|
|
3
|
+
*.py[codz]
|
|
4
|
+
*$py.class
|
|
5
|
+
|
|
6
|
+
# C extensions
|
|
7
|
+
*.so
|
|
8
|
+
|
|
9
|
+
# Distribution / packaging
|
|
10
|
+
.Python
|
|
11
|
+
build/
|
|
12
|
+
develop-eggs/
|
|
13
|
+
dist/
|
|
14
|
+
downloads/
|
|
15
|
+
eggs/
|
|
16
|
+
.eggs/
|
|
17
|
+
lib/
|
|
18
|
+
lib64/
|
|
19
|
+
parts/
|
|
20
|
+
sdist/
|
|
21
|
+
var/
|
|
22
|
+
wheels/
|
|
23
|
+
share/python-wheels/
|
|
24
|
+
*.egg-info/
|
|
25
|
+
.installed.cfg
|
|
26
|
+
*.egg
|
|
27
|
+
MANIFEST
|
|
28
|
+
|
|
29
|
+
# PyInstaller
|
|
30
|
+
# Usually these files are written by a python script from a template
|
|
31
|
+
# before PyInstaller builds the exe, so as to inject date/other infos into it.
|
|
32
|
+
*.manifest
|
|
33
|
+
*.spec
|
|
34
|
+
|
|
35
|
+
# Installer logs
|
|
36
|
+
pip-log.txt
|
|
37
|
+
pip-delete-this-directory.txt
|
|
38
|
+
|
|
39
|
+
# Unit test / coverage reports
|
|
40
|
+
htmlcov/
|
|
41
|
+
.tox/
|
|
42
|
+
.nox/
|
|
43
|
+
.coverage
|
|
44
|
+
.coverage.*
|
|
45
|
+
.cache
|
|
46
|
+
nosetests.xml
|
|
47
|
+
coverage.xml
|
|
48
|
+
*.cover
|
|
49
|
+
*.py.cover
|
|
50
|
+
.hypothesis/
|
|
51
|
+
.pytest_cache/
|
|
52
|
+
cover/
|
|
53
|
+
|
|
54
|
+
# Translations
|
|
55
|
+
*.mo
|
|
56
|
+
*.pot
|
|
57
|
+
|
|
58
|
+
# Django stuff:
|
|
59
|
+
*.log
|
|
60
|
+
local_settings.py
|
|
61
|
+
db.sqlite3
|
|
62
|
+
db.sqlite3-journal
|
|
63
|
+
|
|
64
|
+
# Flask stuff:
|
|
65
|
+
instance/
|
|
66
|
+
.webassets-cache
|
|
67
|
+
|
|
68
|
+
# Scrapy stuff:
|
|
69
|
+
.scrapy
|
|
70
|
+
|
|
71
|
+
# Sphinx documentation
|
|
72
|
+
docs/_build/
|
|
73
|
+
|
|
74
|
+
# PyBuilder
|
|
75
|
+
.pybuilder/
|
|
76
|
+
/target/
|
|
77
|
+
|
|
78
|
+
# Jupyter Notebook
|
|
79
|
+
.ipynb_checkpoints
|
|
80
|
+
|
|
81
|
+
# IPython
|
|
82
|
+
profile_default/
|
|
83
|
+
ipython_config.py
|
|
84
|
+
|
|
85
|
+
# pyenv
|
|
86
|
+
# For a library or package, you might want to ignore these files since the code is
|
|
87
|
+
# intended to run in multiple environments; otherwise, check them in:
|
|
88
|
+
# .python-version
|
|
89
|
+
|
|
90
|
+
# pipenv
|
|
91
|
+
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
|
|
92
|
+
# However, in case of collaboration, if having platform-specific dependencies or dependencies
|
|
93
|
+
# having no cross-platform support, pipenv may install dependencies that don't work, or not
|
|
94
|
+
# install all needed dependencies.
|
|
95
|
+
# Pipfile.lock
|
|
96
|
+
|
|
97
|
+
# UV
|
|
98
|
+
# Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
|
|
99
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
100
|
+
# commonly ignored for libraries.
|
|
101
|
+
# uv.lock
|
|
102
|
+
|
|
103
|
+
# poetry
|
|
104
|
+
# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
|
|
105
|
+
# This is especially recommended for binary packages to ensure reproducibility, and is more
|
|
106
|
+
# commonly ignored for libraries.
|
|
107
|
+
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
|
|
108
|
+
# poetry.lock
|
|
109
|
+
# poetry.toml
|
|
110
|
+
|
|
111
|
+
# pdm
|
|
112
|
+
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
|
|
113
|
+
# pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
|
|
114
|
+
# https://pdm-project.org/en/latest/usage/project/#working-with-version-control
|
|
115
|
+
# pdm.lock
|
|
116
|
+
# pdm.toml
|
|
117
|
+
.pdm-python
|
|
118
|
+
.pdm-build/
|
|
119
|
+
|
|
120
|
+
# pixi
|
|
121
|
+
# Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
|
|
122
|
+
# pixi.lock
|
|
123
|
+
# Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
|
|
124
|
+
# in the .venv directory. It is recommended not to include this directory in version control.
|
|
125
|
+
.pixi
|
|
126
|
+
|
|
127
|
+
# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
|
|
128
|
+
__pypackages__/
|
|
129
|
+
|
|
130
|
+
# Celery stuff
|
|
131
|
+
celerybeat-schedule
|
|
132
|
+
celerybeat.pid
|
|
133
|
+
|
|
134
|
+
# Redis
|
|
135
|
+
*.rdb
|
|
136
|
+
*.aof
|
|
137
|
+
*.pid
|
|
138
|
+
|
|
139
|
+
# RabbitMQ
|
|
140
|
+
mnesia/
|
|
141
|
+
rabbitmq/
|
|
142
|
+
rabbitmq-data/
|
|
143
|
+
|
|
144
|
+
# ActiveMQ
|
|
145
|
+
activemq-data/
|
|
146
|
+
|
|
147
|
+
# SageMath parsed files
|
|
148
|
+
*.sage.py
|
|
149
|
+
|
|
150
|
+
# Environments
|
|
151
|
+
.env
|
|
152
|
+
.envrc
|
|
153
|
+
.venv
|
|
154
|
+
env/
|
|
155
|
+
venv/
|
|
156
|
+
ENV/
|
|
157
|
+
env.bak/
|
|
158
|
+
venv.bak/
|
|
159
|
+
|
|
160
|
+
# Spyder project settings
|
|
161
|
+
.spyderproject
|
|
162
|
+
.spyproject
|
|
163
|
+
|
|
164
|
+
# Rope project settings
|
|
165
|
+
.ropeproject
|
|
166
|
+
|
|
167
|
+
# mkdocs documentation
|
|
168
|
+
/site
|
|
169
|
+
|
|
170
|
+
# mypy
|
|
171
|
+
.mypy_cache/
|
|
172
|
+
.dmypy.json
|
|
173
|
+
dmypy.json
|
|
174
|
+
|
|
175
|
+
# Pyre type checker
|
|
176
|
+
.pyre/
|
|
177
|
+
|
|
178
|
+
# pytype static type analyzer
|
|
179
|
+
.pytype/
|
|
180
|
+
|
|
181
|
+
# Cython debug symbols
|
|
182
|
+
cython_debug/
|
|
183
|
+
|
|
184
|
+
# PyCharm
|
|
185
|
+
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
|
|
186
|
+
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
|
|
187
|
+
# and can be added to the global gitignore or merged into this file. For a more nuclear
|
|
188
|
+
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
|
|
189
|
+
# .idea/
|
|
190
|
+
|
|
191
|
+
# Abstra
|
|
192
|
+
# Abstra is an AI-powered process automation framework.
|
|
193
|
+
# Ignore directories containing user credentials, local state, and settings.
|
|
194
|
+
# Learn more at https://abstra.io/docs
|
|
195
|
+
.abstra/
|
|
196
|
+
|
|
197
|
+
# Visual Studio Code
|
|
198
|
+
# Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore
|
|
199
|
+
# that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
|
|
200
|
+
# and can be added to the global gitignore or merged into this file. However, if you prefer,
|
|
201
|
+
# you could uncomment the following to ignore the entire vscode folder
|
|
202
|
+
# .vscode/
|
|
203
|
+
# Temporary file for partial code execution
|
|
204
|
+
tempCodeRunnerFile.py
|
|
205
|
+
|
|
206
|
+
# Ruff stuff:
|
|
207
|
+
.ruff_cache/
|
|
208
|
+
|
|
209
|
+
# PyPI configuration file
|
|
210
|
+
.pypirc
|
|
211
|
+
|
|
212
|
+
# Marimo
|
|
213
|
+
marimo/_static/
|
|
214
|
+
marimo/_lsp/
|
|
215
|
+
__marimo__/
|
|
216
|
+
|
|
217
|
+
# Streamlit
|
|
218
|
+
.streamlit/secrets.toml
|
|
219
|
+
|
|
220
|
+
.memo
|
|
221
|
+
.tours
|
|
222
|
+
.convert/
|
|
223
|
+
.venv/
|
|
224
|
+
models/
|
|
225
|
+
*.sqlite3
|
yaku-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Shuichi Ohsawa
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
yaku-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: yaku
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Offline translator for command output that keeps identifiers intact
|
|
5
|
+
Project-URL: Homepage, https://github.com/ohsawa0515/yaku
|
|
6
|
+
Project-URL: Repository, https://github.com/ohsawa0515/yaku
|
|
7
|
+
Project-URL: Issues, https://github.com/ohsawa0515/yaku/issues
|
|
8
|
+
Author: Shuichi Ohsawa
|
|
9
|
+
License-Expression: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: cli,nmt,offline,translation
|
|
12
|
+
Classifier: Development Status :: 3 - Alpha
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Natural Language :: Japanese
|
|
16
|
+
Classifier: Operating System :: MacOS
|
|
17
|
+
Classifier: Operating System :: POSIX :: Linux
|
|
18
|
+
Classifier: Programming Language :: Python :: 3
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
22
|
+
Classifier: Topic :: Utilities
|
|
23
|
+
Requires-Python: >=3.12
|
|
24
|
+
Requires-Dist: ctranslate2>=4.0
|
|
25
|
+
Requires-Dist: sentencepiece>=0.2
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
|
|
28
|
+
# yaku
|
|
29
|
+
|
|
30
|
+
English | [日本語](README.ja.md)
|
|
31
|
+
|
|
32
|
+
Fully offline translator for command output. Only the prose gets translated; everything else is left verbatim.
|
|
33
|
+
|
|
34
|
+
Identifiers, flags, paths and quoted strings survive unchanged, because they are moved out of the way before the model sees the text and put back afterwards. No network access, no API keys, no data leaving the machine.
|
|
35
|
+
|
|
36
|
+
## Install
|
|
37
|
+
|
|
38
|
+
```console
|
|
39
|
+
pip install yaku
|
|
40
|
+
yaku setup --lang ja
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
`yaku setup` downloads [FuguMT](https://huggingface.co/staka/fugumt-en-ja) at a pinned revision and converts it to a 62 MB int8 model. The model is not bundled because it is CC BY-SA 4.0. The converter needs torch, which yaku does not depend on, so the conversion runs in a throwaway environment created with [uv](https://docs.astral.sh/uv/).
|
|
44
|
+
|
|
45
|
+
`yaku setup` requires uv. Install it and put it on `PATH` before running the command.
|
|
46
|
+
|
|
47
|
+
## Use
|
|
48
|
+
|
|
49
|
+
```console
|
|
50
|
+
terraform plan 2>&1 | yaku
|
|
51
|
+
man tar | col -bx | yaku
|
|
52
|
+
kubectl describe pod web | yaku
|
|
53
|
+
pbpaste | yaku
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Useful options:
|
|
57
|
+
|
|
58
|
+
| Option | Effect |
|
|
59
|
+
| --- | --- |
|
|
60
|
+
| `--lang ja` | target language, which also selects the model and thresholds |
|
|
61
|
+
| `--fallback both\|original\|mark` | what to emit for a line that fails the quality check |
|
|
62
|
+
| `--show-reason` | append why a line failed |
|
|
63
|
+
| `--stats` | write a summary to stderr |
|
|
64
|
+
| `--dry-run` | show classification and masking without loading a model |
|
|
65
|
+
| `--join` / `--no-join` | override the automatic wrapped-line detection |
|
|
66
|
+
| `--width N` | wrap the translated text |
|
|
67
|
+
| `--mask-off NAME,...` | disable mask rules |
|
|
68
|
+
|
|
69
|
+
## How it works
|
|
70
|
+
|
|
71
|
+
```
|
|
72
|
+
stdin
|
|
73
|
+
↓ strip ANSI escapes
|
|
74
|
+
↓ analysis pass decides whether lines are wrapped prose
|
|
75
|
+
↓ classify pass through / command example / translatable
|
|
76
|
+
↓ join reassemble wrapped sentences
|
|
77
|
+
↓ mask identifiers become T1, T2, ...
|
|
78
|
+
↓ split back into sentences
|
|
79
|
+
↓ cache keyed on the normalised masked text
|
|
80
|
+
↓ translate CTranslate2 + SentencePiece
|
|
81
|
+
↓ check lost placeholders, length ratio, repetition
|
|
82
|
+
↓ unmask identifiers restored
|
|
83
|
+
stdout
|
|
84
|
+
```
|
|
85
|
+
|
|
86
|
+
Two parts carry most of the value.
|
|
87
|
+
|
|
88
|
+
**Masking** is not about withholding text. A sequence model that has never seen `aws_instance.web_server_01` in training will hallucinate around it; `T1 will be created` is ordinary English and translates cleanly. Masking also cuts the request by 5 to 10 percent in tokens, and a placeholder that fails to come back is the only reliable sign that a clause was dropped.
|
|
89
|
+
|
|
90
|
+
**The quality check** exists because dropped clauses cannot be prevented. Each line is verified afterwards, and anything suspect is shown with its original text rather than silently replaced.
|
|
91
|
+
|
|
92
|
+
## Limits
|
|
93
|
+
|
|
94
|
+
- Translation quality is bounded by a 60M parameter model. It will not match a large language model.
|
|
95
|
+
- Dropped clauses cannot be eliminated, only detected. Roughly 5 to 9 percent of translated lines in manual pages are flagged, of which some are real drops and some are Japanese simply being more compact than the ratio check allows.
|
|
96
|
+
- Whether lines are joined is decided once for the whole input. Inside record-per-line output, a paragraph is still rejoined when its lines are filled to the wrap column, but a wrap that lands just before a capitalised word is left split.
|
|
97
|
+
- Japanese output sometimes loses a space around a restored identifier.
|
|
98
|
+
|
|
99
|
+
## Adding a language
|
|
100
|
+
|
|
101
|
+
`--lang` switches the model and the measured constants together, so a new language needs a profile, not just a model.
|
|
102
|
+
|
|
103
|
+
1. Pick a CTranslate2-compatible model (Marian and OPUS-MT convert cleanly).
|
|
104
|
+
2. `uv run tools/check_placeholders.py` to choose a placeholder format.
|
|
105
|
+
3. `uv run tools/eval_translate.py` to measure the length ratio bounds.
|
|
106
|
+
4. Write `src/yaku/rules/target/<lang>.toml`.
|
|
107
|
+
|
|
108
|
+
## Development
|
|
109
|
+
|
|
110
|
+
```console
|
|
111
|
+
uv sync
|
|
112
|
+
export YAKU_MODELS_DIR=models # use the checkout's model, not the installed one
|
|
113
|
+
uv run pytest
|
|
114
|
+
uv run ruff check src tests tools && uv run ruff format src tests tools
|
|
115
|
+
uv run tools/measure_wrap.py captured-output.txt
|
|
116
|
+
uv run tools/eval_translate.py captured-output.txt
|
|
117
|
+
```
|
|
118
|
+
|
|
119
|
+
Rules live in `src/yaku/rules/` as TOML. Users can add their own in `~/.config/yaku/rules/` or point `--rules-dir` at a directory; shipped rules are never edited in place.
|
|
120
|
+
|
|
121
|
+
## Licence
|
|
122
|
+
|
|
123
|
+
MIT. The default model, [FuguMT](https://huggingface.co/staka/fugumt-en-ja), is CC BY-SA 4.0 and is downloaded separately.
|
yaku-0.1.0/README.ja.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# yaku
|
|
2
|
+
|
|
3
|
+
[English](README.md) | 日本語
|
|
4
|
+
|
|
5
|
+
コマンド出力を翻訳する、完全オフラインのフィルタです。翻訳するのは文章の部分だけで、それ以外は原文のまま残します。
|
|
6
|
+
|
|
7
|
+
識別子、フラグ、パス、引用符で囲まれた文字列は変わりません。モデルに渡す前に退避し、翻訳後に元へ戻すためです。ネットワーク通信も API キーも不要で、データがマシンの外に出ることはありません。
|
|
8
|
+
|
|
9
|
+
## インストール
|
|
10
|
+
|
|
11
|
+
```console
|
|
12
|
+
pip install yaku
|
|
13
|
+
yaku setup --lang ja
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`yaku setup` は [FuguMT](https://huggingface.co/staka/fugumt-en-ja) を固定したリビジョンで取得し、62 MB の int8 モデルに変換します。モデルは CC BY-SA 4.0 のため、同梱していません。変換には torch が必要ですが、yaku の依存には含めていません。そのため変換は使い捨ての環境で行い、その作成に [uv](https://docs.astral.sh/uv/) を使います。
|
|
17
|
+
|
|
18
|
+
`yaku setup` の実行には uv が必要です。uv を `PATH` 上に入れてから実行してください。
|
|
19
|
+
|
|
20
|
+
## 使い方
|
|
21
|
+
|
|
22
|
+
```console
|
|
23
|
+
terraform plan 2>&1 | yaku
|
|
24
|
+
man tar | col -bx | yaku
|
|
25
|
+
kubectl describe pod web | yaku
|
|
26
|
+
pbpaste | yaku
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
主なオプションは次のとおりです。
|
|
30
|
+
|
|
31
|
+
| オプション | 内容 |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| `--lang ja` | 翻訳先の言語です。モデルと閾値も、これに合わせて切り替わります |
|
|
34
|
+
| `--fallback both\|original\|mark` | 品質検査に通らなかった行の出力方法です |
|
|
35
|
+
| `--show-reason` | 検査に通らなかった理由を付けます |
|
|
36
|
+
| `--stats` | 集計を stderr に出力します |
|
|
37
|
+
| `--dry-run` | モデルを読み込まず、行の分類とマスクの結果を表示します |
|
|
38
|
+
| `--join` / `--no-join` | 折り返し行の自動判定を上書きします |
|
|
39
|
+
| `--width N` | 翻訳結果を指定の幅で折り返します |
|
|
40
|
+
| `--mask-off NAME,...` | マスク規則を無効にします |
|
|
41
|
+
|
|
42
|
+
## 仕組み
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
stdin
|
|
46
|
+
↓ ANSI エスケープの除去
|
|
47
|
+
↓ 解析パス 折り返された文章かどうかを判定
|
|
48
|
+
↓ 分類 そのまま出力 / コマンド例 / 翻訳対象
|
|
49
|
+
↓ 結合 折り返された文を再構成
|
|
50
|
+
↓ マスク 識別子を T1, T2, ... に置き換え
|
|
51
|
+
↓ 分割 文単位に戻す
|
|
52
|
+
↓ キャッシュ 正規化したマスク後の文をキーにする
|
|
53
|
+
↓ 翻訳 CTranslate2 + SentencePiece
|
|
54
|
+
↓ 検査 プレースホルダの消失、長さ比、反復
|
|
55
|
+
↓ 復元 識別子を元に戻す
|
|
56
|
+
stdout
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
価値の大部分は次の 2 つにあります。
|
|
60
|
+
|
|
61
|
+
マスクは、テキストをモデルに見せないためのものではありません。`aws_instance.web_server_01` のような文字列は、系列変換モデルが学習時に見ていない形なので、その周辺で誤った出力を生みます。`T1 will be created` は通常の英文なので、きれいに訳せます。マスクによってモデルへの入力はトークン数で 5 から 10 パーセント減り、プレースホルダが戻ってこないことが、節が訳抜けしたことを確実に示す唯一の手がかりになります。
|
|
62
|
+
|
|
63
|
+
品質検査があるのは、訳抜けを防げないからです。各行を翻訳後に検査し、疑わしいものは通知なく置き換えず、原文を併記して表示します。
|
|
64
|
+
|
|
65
|
+
## 制約
|
|
66
|
+
|
|
67
|
+
- 訳質は 60M パラメータのモデルの範囲にとどまり、大規模言語モデルには及びません。
|
|
68
|
+
- 訳抜けはなくせず、検出できるだけです。man ページでは翻訳した行の 5 から 9 パーセントほどが検査に引っかかります。その中には、本当に訳が抜けたものと、日本語が長さ比の想定より短くなっただけのものがあります。
|
|
69
|
+
- 行を結合するかどうかは、入力全体に対して 1 回だけ判定します。1 行 1 記録の出力の中でも、折り返し幅まで埋まった行からなる段落は再結合しますが、折り返し位置の直後が大文字で始まる語の場合は分かれたままになります。
|
|
70
|
+
- 日本語の出力で、復元した識別子の前後の空白が欠けることがあります。
|
|
71
|
+
|
|
72
|
+
## 言語の追加
|
|
73
|
+
|
|
74
|
+
`--lang` はモデルと、計測した定数をまとめて切り替えます。新しい言語には、モデルだけでなくプロファイルが必要です。
|
|
75
|
+
|
|
76
|
+
1. CTranslate2 に変換できるモデルを選びます(Marian と OPUS-MT は問題なく変換できます)。
|
|
77
|
+
2. `uv run tools/check_placeholders.py` でプレースホルダの形式を選びます。
|
|
78
|
+
3. `uv run tools/eval_translate.py` で長さ比の範囲を計測します。
|
|
79
|
+
4. `src/yaku/rules/target/<lang>.toml` を書きます。
|
|
80
|
+
|
|
81
|
+
## 開発
|
|
82
|
+
|
|
83
|
+
```console
|
|
84
|
+
uv sync
|
|
85
|
+
export YAKU_MODELS_DIR=models # インストール済みではなく、チェックアウト内のモデルを使う
|
|
86
|
+
uv run pytest
|
|
87
|
+
uv run ruff check src tests tools && uv run ruff format src tests tools
|
|
88
|
+
uv run tools/measure_wrap.py captured-output.txt
|
|
89
|
+
uv run tools/eval_translate.py captured-output.txt
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
規則は `src/yaku/rules/` に TOML で置いています。独自の規則は `~/.config/yaku/rules/` に追加するか、`--rules-dir` でディレクトリを指定します。同梱の規則を直接編集することはありません。
|
|
93
|
+
|
|
94
|
+
## ライセンス
|
|
95
|
+
|
|
96
|
+
MIT です。標準のモデルである [FuguMT](https://huggingface.co/staka/fugumt-en-ja) は CC BY-SA 4.0 で、別途ダウンロードします。
|
yaku-0.1.0/README.md
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
# yaku
|
|
2
|
+
|
|
3
|
+
English | [日本語](README.ja.md)
|
|
4
|
+
|
|
5
|
+
Fully offline translator for command output. Only the prose gets translated; everything else is left verbatim.
|
|
6
|
+
|
|
7
|
+
Identifiers, flags, paths and quoted strings survive unchanged, because they are moved out of the way before the model sees the text and put back afterwards. No network access, no API keys, no data leaving the machine.
|
|
8
|
+
|
|
9
|
+
## Install
|
|
10
|
+
|
|
11
|
+
```console
|
|
12
|
+
pip install yaku
|
|
13
|
+
yaku setup --lang ja
|
|
14
|
+
```
|
|
15
|
+
|
|
16
|
+
`yaku setup` downloads [FuguMT](https://huggingface.co/staka/fugumt-en-ja) at a pinned revision and converts it to a 62 MB int8 model. The model is not bundled because it is CC BY-SA 4.0. The converter needs torch, which yaku does not depend on, so the conversion runs in a throwaway environment created with [uv](https://docs.astral.sh/uv/).
|
|
17
|
+
|
|
18
|
+
`yaku setup` requires uv. Install it and put it on `PATH` before running the command.
|
|
19
|
+
|
|
20
|
+
## Use
|
|
21
|
+
|
|
22
|
+
```console
|
|
23
|
+
terraform plan 2>&1 | yaku
|
|
24
|
+
man tar | col -bx | yaku
|
|
25
|
+
kubectl describe pod web | yaku
|
|
26
|
+
pbpaste | yaku
|
|
27
|
+
```
|
|
28
|
+
|
|
29
|
+
Useful options:
|
|
30
|
+
|
|
31
|
+
| Option | Effect |
|
|
32
|
+
| --- | --- |
|
|
33
|
+
| `--lang ja` | target language, which also selects the model and thresholds |
|
|
34
|
+
| `--fallback both\|original\|mark` | what to emit for a line that fails the quality check |
|
|
35
|
+
| `--show-reason` | append why a line failed |
|
|
36
|
+
| `--stats` | write a summary to stderr |
|
|
37
|
+
| `--dry-run` | show classification and masking without loading a model |
|
|
38
|
+
| `--join` / `--no-join` | override the automatic wrapped-line detection |
|
|
39
|
+
| `--width N` | wrap the translated text |
|
|
40
|
+
| `--mask-off NAME,...` | disable mask rules |
|
|
41
|
+
|
|
42
|
+
## How it works
|
|
43
|
+
|
|
44
|
+
```
|
|
45
|
+
stdin
|
|
46
|
+
↓ strip ANSI escapes
|
|
47
|
+
↓ analysis pass decides whether lines are wrapped prose
|
|
48
|
+
↓ classify pass through / command example / translatable
|
|
49
|
+
↓ join reassemble wrapped sentences
|
|
50
|
+
↓ mask identifiers become T1, T2, ...
|
|
51
|
+
↓ split back into sentences
|
|
52
|
+
↓ cache keyed on the normalised masked text
|
|
53
|
+
↓ translate CTranslate2 + SentencePiece
|
|
54
|
+
↓ check lost placeholders, length ratio, repetition
|
|
55
|
+
↓ unmask identifiers restored
|
|
56
|
+
stdout
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Two parts carry most of the value.
|
|
60
|
+
|
|
61
|
+
**Masking** is not about withholding text. A sequence model that has never seen `aws_instance.web_server_01` in training will hallucinate around it; `T1 will be created` is ordinary English and translates cleanly. Masking also cuts the request by 5 to 10 percent in tokens, and a placeholder that fails to come back is the only reliable sign that a clause was dropped.
|
|
62
|
+
|
|
63
|
+
**The quality check** exists because dropped clauses cannot be prevented. Each line is verified afterwards, and anything suspect is shown with its original text rather than silently replaced.
|
|
64
|
+
|
|
65
|
+
## Limits
|
|
66
|
+
|
|
67
|
+
- Translation quality is bounded by a 60M parameter model. It will not match a large language model.
|
|
68
|
+
- Dropped clauses cannot be eliminated, only detected. Roughly 5 to 9 percent of translated lines in manual pages are flagged, of which some are real drops and some are Japanese simply being more compact than the ratio check allows.
|
|
69
|
+
- Whether lines are joined is decided once for the whole input. Inside record-per-line output, a paragraph is still rejoined when its lines are filled to the wrap column, but a wrap that lands just before a capitalised word is left split.
|
|
70
|
+
- Japanese output sometimes loses a space around a restored identifier.
|
|
71
|
+
|
|
72
|
+
## Adding a language
|
|
73
|
+
|
|
74
|
+
`--lang` switches the model and the measured constants together, so a new language needs a profile, not just a model.
|
|
75
|
+
|
|
76
|
+
1. Pick a CTranslate2-compatible model (Marian and OPUS-MT convert cleanly).
|
|
77
|
+
2. `uv run tools/check_placeholders.py` to choose a placeholder format.
|
|
78
|
+
3. `uv run tools/eval_translate.py` to measure the length ratio bounds.
|
|
79
|
+
4. Write `src/yaku/rules/target/<lang>.toml`.
|
|
80
|
+
|
|
81
|
+
## Development
|
|
82
|
+
|
|
83
|
+
```console
|
|
84
|
+
uv sync
|
|
85
|
+
export YAKU_MODELS_DIR=models # use the checkout's model, not the installed one
|
|
86
|
+
uv run pytest
|
|
87
|
+
uv run ruff check src tests tools && uv run ruff format src tests tools
|
|
88
|
+
uv run tools/measure_wrap.py captured-output.txt
|
|
89
|
+
uv run tools/eval_translate.py captured-output.txt
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
Rules live in `src/yaku/rules/` as TOML. Users can add their own in `~/.config/yaku/rules/` or point `--rules-dir` at a directory; shipped rules are never edited in place.
|
|
93
|
+
|
|
94
|
+
## Licence
|
|
95
|
+
|
|
96
|
+
MIT. The default model, [FuguMT](https://huggingface.co/staka/fugumt-en-ja), is CC BY-SA 4.0 and is downloaded separately.
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["hatchling"]
|
|
3
|
+
build-backend = "hatchling.build"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "yaku"
|
|
7
|
+
description = "Offline translator for command output that keeps identifiers intact"
|
|
8
|
+
readme = "README.md"
|
|
9
|
+
requires-python = ">=3.12"
|
|
10
|
+
license = "MIT"
|
|
11
|
+
license-files = ["LICENSE"]
|
|
12
|
+
authors = [{ name = "Shuichi Ohsawa" }]
|
|
13
|
+
keywords = ["translation", "cli", "offline", "nmt"]
|
|
14
|
+
classifiers = [
|
|
15
|
+
"Development Status :: 3 - Alpha",
|
|
16
|
+
"Environment :: Console",
|
|
17
|
+
"Intended Audience :: Developers",
|
|
18
|
+
"Natural Language :: Japanese",
|
|
19
|
+
"Operating System :: MacOS",
|
|
20
|
+
"Operating System :: POSIX :: Linux",
|
|
21
|
+
"Programming Language :: Python :: 3",
|
|
22
|
+
"Programming Language :: Python :: 3.12",
|
|
23
|
+
"Programming Language :: Python :: 3.13",
|
|
24
|
+
"Topic :: Text Processing :: Linguistic",
|
|
25
|
+
"Topic :: Utilities",
|
|
26
|
+
]
|
|
27
|
+
# Rule files are TOML: tomllib is in the standard library, and its literal
|
|
28
|
+
# strings hold regular expressions without escaping.
|
|
29
|
+
dependencies = [
|
|
30
|
+
"ctranslate2>=4.0",
|
|
31
|
+
"sentencepiece>=0.2",
|
|
32
|
+
]
|
|
33
|
+
dynamic = ["version"]
|
|
34
|
+
|
|
35
|
+
[project.urls]
|
|
36
|
+
Homepage = "https://github.com/ohsawa0515/yaku"
|
|
37
|
+
Repository = "https://github.com/ohsawa0515/yaku"
|
|
38
|
+
Issues = "https://github.com/ohsawa0515/yaku/issues"
|
|
39
|
+
|
|
40
|
+
[project.scripts]
|
|
41
|
+
yaku = "yaku.cli:run"
|
|
42
|
+
|
|
43
|
+
[dependency-groups]
|
|
44
|
+
dev = ["pytest>=8", "ruff>=0.8"]
|
|
45
|
+
|
|
46
|
+
[tool.hatch.version]
|
|
47
|
+
path = "src/yaku/__init__.py"
|
|
48
|
+
|
|
49
|
+
# An allowlist, so local scratch files ignored outside .gitignore stay out.
|
|
50
|
+
[tool.hatch.build.targets.sdist]
|
|
51
|
+
only-include = ["src", "tests", "LICENSE", "README.md", "README.ja.md", "pyproject.toml"]
|
|
52
|
+
|
|
53
|
+
[tool.hatch.build.targets.wheel]
|
|
54
|
+
packages = ["src/yaku"]
|
|
55
|
+
|
|
56
|
+
[tool.pytest.ini_options]
|
|
57
|
+
testpaths = ["tests"]
|
|
58
|
+
|
|
59
|
+
[tool.ruff]
|
|
60
|
+
line-length = 100
|
|
61
|
+
target-version = "py312"
|
|
62
|
+
|
|
63
|
+
[tool.ruff.lint]
|
|
64
|
+
select = ["E", "F", "W", "I", "B", "UP", "SIM", "RUF"]
|
|
65
|
+
ignore = [
|
|
66
|
+
"RUF001", # ambiguous characters are intended: box drawing, fullwidth punctuation
|
|
67
|
+
"RUF003",
|
|
68
|
+
]
|
|
69
|
+
|
|
70
|
+
[tool.ruff.lint.per-file-ignores]
|
|
71
|
+
# Test inputs are real command output, which is as long as it is.
|
|
72
|
+
"tests/*" = ["E501"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.1.0"
|