mojiblame 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- mojiblame-0.1.0/.github/workflows/publish.yml +26 -0
- mojiblame-0.1.0/.github/workflows/test.yml +38 -0
- mojiblame-0.1.0/.gitignore +15 -0
- mojiblame-0.1.0/.mojiblameignore +4 -0
- mojiblame-0.1.0/.pre-commit-hooks.yaml +8 -0
- mojiblame-0.1.0/LICENSE +21 -0
- mojiblame-0.1.0/PKG-INFO +249 -0
- mojiblame-0.1.0/README.md +220 -0
- mojiblame-0.1.0/docs/assets/blame-demo.gif +0 -0
- mojiblame-0.1.0/docs/assets/hero.png +0 -0
- mojiblame-0.1.0/docs/assets/pipeline.svg +77 -0
- mojiblame-0.1.0/pyproject.toml +50 -0
- mojiblame-0.1.0/src/mojiblame/__init__.py +3 -0
- mojiblame-0.1.0/src/mojiblame/cli.py +259 -0
- mojiblame-0.1.0/src/mojiblame/detect/__init__.py +0 -0
- mojiblame-0.1.0/src/mojiblame/detect/codecs.py +66 -0
- mojiblame-0.1.0/src/mojiblame/detect/facts.py +104 -0
- mojiblame-0.1.0/src/mojiblame/detect/mojibake.py +114 -0
- mojiblame-0.1.0/src/mojiblame/fixing/__init__.py +0 -0
- mojiblame-0.1.0/src/mojiblame/fixing/apply.py +51 -0
- mojiblame-0.1.0/src/mojiblame/fixing/journal.py +79 -0
- mojiblame-0.1.0/src/mojiblame/gitfx/__init__.py +0 -0
- mojiblame-0.1.0/src/mojiblame/gitfx/forensics.py +197 -0
- mojiblame-0.1.0/src/mojiblame/gitfx/recover.py +85 -0
- mojiblame-0.1.0/src/mojiblame/gitfx/repo.py +125 -0
- mojiblame-0.1.0/src/mojiblame/hooks/__init__.py +0 -0
- mojiblame-0.1.0/src/mojiblame/hooks/agent.py +93 -0
- mojiblame-0.1.0/src/mojiblame/hooks/precommit.py +39 -0
- mojiblame-0.1.0/src/mojiblame/ignore.py +63 -0
- mojiblame-0.1.0/src/mojiblame/model.py +127 -0
- mojiblame-0.1.0/src/mojiblame/report/__init__.py +0 -0
- mojiblame-0.1.0/src/mojiblame/report/render.py +158 -0
- mojiblame-0.1.0/src/mojiblame/scanner.py +178 -0
- mojiblame-0.1.0/tests/__init__.py +0 -0
- mojiblame-0.1.0/tests/conftest.py +77 -0
- mojiblame-0.1.0/tests/test_detect.py +119 -0
- mojiblame-0.1.0/tests/test_forensics.py +141 -0
- mojiblame-0.1.0/tests/test_hooks_and_cli.py +178 -0
- mojiblame-0.1.0/tests/test_ignore.py +61 -0
- mojiblame-0.1.0/tests/test_scanner_and_fix.py +100 -0
- mojiblame-0.1.0/uv.lock +419 -0
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
name: publish
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags: ["v*"]
|
|
6
|
+
|
|
7
|
+
jobs:
|
|
8
|
+
publish:
|
|
9
|
+
runs-on: ubuntu-latest
|
|
10
|
+
environment: pypi
|
|
11
|
+
permissions:
|
|
12
|
+
# Trusted publishing: PyPI verifies this workflow's identity directly, so
|
|
13
|
+
# there is no long-lived API token sitting in repository secrets.
|
|
14
|
+
id-token: write
|
|
15
|
+
|
|
16
|
+
steps:
|
|
17
|
+
- uses: actions/checkout@v4
|
|
18
|
+
- uses: astral-sh/setup-uv@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: "3.12"
|
|
21
|
+
|
|
22
|
+
- run: uv sync --all-extras --dev
|
|
23
|
+
- run: uv run python -m pytest -q
|
|
24
|
+
- run: uv build
|
|
25
|
+
- name: Publish to PyPI
|
|
26
|
+
run: uv publish --trusted-publishing always
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
name: test
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
branches: [master, main]
|
|
6
|
+
pull_request:
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
test:
|
|
10
|
+
# Windows is not an afterthought here - it is where most of this damage is
|
|
11
|
+
# created, so a green Windows job is the point rather than a bonus.
|
|
12
|
+
runs-on: ${{ matrix.os }}
|
|
13
|
+
strategy:
|
|
14
|
+
fail-fast: false
|
|
15
|
+
matrix:
|
|
16
|
+
os: [ubuntu-latest, windows-latest, macos-latest]
|
|
17
|
+
python-version: ["3.10", "3.11", "3.12", "3.13"]
|
|
18
|
+
|
|
19
|
+
steps:
|
|
20
|
+
- uses: actions/checkout@v4
|
|
21
|
+
|
|
22
|
+
- name: Install uv
|
|
23
|
+
uses: astral-sh/setup-uv@v5
|
|
24
|
+
with:
|
|
25
|
+
python-version: ${{ matrix.python-version }}
|
|
26
|
+
|
|
27
|
+
- name: Configure git for the synthetic-repo tests
|
|
28
|
+
run: |
|
|
29
|
+
git config --global user.email "ci@example.invalid"
|
|
30
|
+
git config --global user.name "ci"
|
|
31
|
+
git config --global init.defaultBranch main
|
|
32
|
+
|
|
33
|
+
- run: uv sync --all-extras --dev
|
|
34
|
+
|
|
35
|
+
- run: uv run python -m pytest -q
|
|
36
|
+
|
|
37
|
+
- name: Dogfood - the tool must find nothing in its own source
|
|
38
|
+
run: uv run mojiblame scan .
|
mojiblame-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Ozan
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
mojiblame-0.1.0/PKG-INFO
ADDED
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: mojiblame
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Git-aware encoding forensics: find which commit corrupted your text, prove every fix byte-exact, recover what other tools call unrecoverable.
|
|
5
|
+
Project-URL: Homepage, https://github.com/Golemozan/mojiblame
|
|
6
|
+
Project-URL: Repository, https://github.com/Golemozan/mojiblame
|
|
7
|
+
Project-URL: Issues, https://github.com/Golemozan/mojiblame/issues
|
|
8
|
+
Author: Ozan
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: cp1252,encoding,forensics,git,mojibake,pre-commit,utf-8,windows
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Environment :: Console
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Software Development :: Quality Assurance
|
|
22
|
+
Classifier: Topic :: Text Processing :: General
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Requires-Dist: charset-normalizer>=3.3
|
|
25
|
+
Requires-Dist: ftfy>=6.2
|
|
26
|
+
Requires-Dist: rich>=13.7
|
|
27
|
+
Requires-Dist: typer>=0.12
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# mojiblame
|
|
31
|
+
|
|
32
|
+
<!-- mojiblame:ignore-file — this README intentionally contains damaged encoding examples. -->
|
|
33
|
+
|
|
34
|
+
**Git-aware encoding forensics.** Find the commit that corrupted your text, prove every fix
|
|
35
|
+
byte-exact, and recover characters other tools call unrecoverable.
|
|
36
|
+
|
|
37
|
+
[](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml)
|
|
38
|
+
[](https://pypi.org/project/mojiblame/)
|
|
39
|
+
[](https://pypi.org/project/mojiblame/)
|
|
40
|
+
[](LICENSE)
|
|
41
|
+
|
|
42
|
+

|
|
43
|
+
|
|
44
|
+
```console
|
|
45
|
+
$ mojiblame scan .
|
|
46
|
+
|
|
47
|
+
damage proof file detail
|
|
48
|
+
lossy unprovable docs/kurulum.md 19 character(s) destroyed (U+FFFD) - try: mojiblame blame
|
|
49
|
+
mojibake byte-exact src/tr.json latin-1 -> utf-8, 1 layer(s), reversal verified byte-exact
|
|
50
|
+
|
|
51
|
+
2 scanned · 2 damaged · 1 fixable now · 1 needs history (mojiblame blame <file>)
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## The problem
|
|
57
|
+
|
|
58
|
+
AI coding agents read and write your files through shells and subprocesses. On Windows —
|
|
59
|
+
and anywhere the default codepage is not UTF-8 — that pipeline silently mangles non-ASCII
|
|
60
|
+
text. Nobody notices, because nothing fails: the agent writes `ö` instead of `ö`, the patch
|
|
61
|
+
context stops matching, a hook prints garbage, and the damage sits in the repo for weeks.
|
|
62
|
+
|
|
63
|
+
This is not hypothetical. It has been reported against
|
|
64
|
+
[Codex on Windows](https://github.com/openai/codex/issues/4131) — Turkish characters coming
|
|
65
|
+
back wrong from `Get-Content`/`Set-Content` — and again as
|
|
66
|
+
[mojibake breaking patch application](https://community.openai.com/t/codex-on-windows-utf-8-mojibake-and-patch-context-mismatch-issues/1384116),
|
|
67
|
+
where the agent can no longer edit a file because the bytes it read are not the bytes on disk.
|
|
68
|
+
|
|
69
|
+
Existing tools look at a file **as it is now** and **guess**. `ftfy` is an excellent library
|
|
70
|
+
for repairing a string, but it does not scan a repository, does not know your git history,
|
|
71
|
+
and cannot tell you whether its repair is correct or merely plausible.
|
|
72
|
+
|
|
73
|
+

|
|
74
|
+
|
|
75
|
+
## What mojiblame does differently
|
|
76
|
+
|
|
77
|
+
### 1. It separates three kinds of damage that other tools blur together
|
|
78
|
+
|
|
79
|
+
| | What happened | Fix | Standard of proof |
|
|
80
|
+
|---|---|---|---|
|
|
81
|
+
| **wrong-codec** | The bytes are not UTF-8 at all (cp1254, cp1251, latin-1…). The text is intact; only the codec is wrong. | transcode | **reversible** — re-encoding restores the original bytes. The codec itself is inferred, and we say so. |
|
|
82
|
+
| **mojibake** | The file *is* valid UTF-8, but the characters are wrong: UTF-8 was decoded as a legacy codepage and re-encoded. `ö` → `ö`. Stacks to several layers. | repair | **byte-exact** — see below |
|
|
83
|
+
| **lossy** | `U+FFFD` replacement characters. The bytes were destroyed at decode time. They are not in the file any more. | *none* | **unprovable from the file** — only history can help |
|
|
84
|
+
|
|
85
|
+
The distinction is not academic. A tool that treats the third case like the first will happily
|
|
86
|
+
"fix" a file and hand you plausible fiction.
|
|
87
|
+
|
|
88
|
+
### 2. Every fix is proven, or it is not applied
|
|
89
|
+
|
|
90
|
+
For mojibake we take the repair plan, **invert it**, and re-apply the inversion to the repaired
|
|
91
|
+
text. If that reproduces the bytes currently on disk exactly, the repair is not a guess: the
|
|
92
|
+
recovered text is a verified preimage of the file as it stands, not the most plausible-looking
|
|
93
|
+
candidate.
|
|
94
|
+
|
|
95
|
+
If the inversion does not reproduce them, mojiblame reports the finding and **refuses to write**:
|
|
96
|
+
|
|
97
|
+
```console
|
|
98
|
+
mojibake unprovable README.md looks like partially decoded UTF-8 mojibake,
|
|
99
|
+
but the reversal is not exact - not fixing
|
|
100
|
+
```
|
|
101
|
+
|
|
102
|
+
No proof, no write. That rule has no exceptions.
|
|
103
|
+
|
|
104
|
+
### 3. It recovers what the file alone cannot
|
|
105
|
+
|
|
106
|
+
`U+FFFD` means the bytes are gone, and no amount of cleverness gets them back out of the file.
|
|
107
|
+
But git kept every earlier version. `mojiblame blame` walks that history, finds the commit where
|
|
108
|
+
the damage appeared, works out which decode step caused it, and restores the destroyed
|
|
109
|
+
characters from the last clean version:
|
|
110
|
+
|
|
111
|
+
```console
|
|
112
|
+
$ mojiblame blame docs/kurulum.md
|
|
113
|
+
|
|
114
|
+
kurulum.md
|
|
115
|
+
|
|
116
|
+
19 character(s) destroyed (U+FFFD)
|
|
117
|
+
corruption entered 03bbd41 2026-09-02 chore: toplu yeniden bicimlendirme
|
|
118
|
+
last clean version c6455a1 2026-09-02 docs: kurulum notlari eklendi
|
|
119
|
+
transform cp1254 -> utf-8
|
|
120
|
+
recoverable 19 / 19 (proof: re-applying the transform reproduces this file exactly)
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
That last line is the point. We reconstruct the corruption, apply it to the recovered text, and
|
|
124
|
+
check it reproduces today's file byte for byte. If it does, the recovery is a deduction, not a
|
|
125
|
+
best guess — and mojiblame says which one it is either way.
|
|
126
|
+
|
|
127
|
+
**It splices, it does not revert.** `git checkout <old-sha> -- file` would throw away every
|
|
128
|
+
legitimate edit made since the damage. mojiblame keeps today's file and replaces only the runs
|
|
129
|
+
of destroyed characters.
|
|
130
|
+
|
|
131
|
+

|
|
132
|
+
|
|
133
|
+
## Install
|
|
134
|
+
|
|
135
|
+
```console
|
|
136
|
+
uv tool install mojiblame # or: pipx install mojiblame
|
|
137
|
+
```
|
|
138
|
+
|
|
139
|
+
## Use
|
|
140
|
+
|
|
141
|
+
```console
|
|
142
|
+
mojiblame scan [PATH] read-only report, with the standard of proof for each finding
|
|
143
|
+
mojiblame blame <FILE> which commit broke it, which step, what history can restore
|
|
144
|
+
mojiblame fix [PATH] apply only the provable fixes; --dry-run, --yes
|
|
145
|
+
mojiblame undo restore the originals from the last fix
|
|
146
|
+
mojiblame guard [PATH] exit non-zero if anything is corrupted (CI, hooks)
|
|
147
|
+
mojiblame init wire it into pre-commit, GitHub Actions, or your agent
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
`fix` journals the original bytes before it changes anything, so `undo` is byte-identical —
|
|
151
|
+
not "close enough". Inside a git repo, scanning honours `.gitignore`.
|
|
152
|
+
|
|
153
|
+
## Wire it in
|
|
154
|
+
|
|
155
|
+
**pre-commit** — the version most people should use, because it runs without anyone remembering it:
|
|
156
|
+
|
|
157
|
+
```yaml
|
|
158
|
+
repos:
|
|
159
|
+
- repo: https://github.com/Golemozan/mojiblame
|
|
160
|
+
rev: v0.1.0
|
|
161
|
+
hooks:
|
|
162
|
+
- id: mojiblame
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
**GitHub Actions** — `mojiblame init --github`, or:
|
|
166
|
+
|
|
167
|
+
```yaml
|
|
168
|
+
- run: pip install mojiblame
|
|
169
|
+
- run: mojiblame guard .
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
**Coding agents** — `mojiblame init --claude` installs a `PreToolUse` hook that inspects writes
|
|
173
|
+
*before they reach the disk* and blocks the ones that would introduce corruption. Cleaning up
|
|
174
|
+
afterwards is the losing half of this problem; the agent is told what it was about to corrupt
|
|
175
|
+
and can retry correctly:
|
|
176
|
+
|
|
177
|
+
```
|
|
178
|
+
mojiblame blocked this write to notes.md: mojibake (cp1254 -> utf-8, 1 layer(s)).
|
|
179
|
+
The text was decoded with the wrong codec somewhere upstream.
|
|
180
|
+
Intended text starts: 'yönetici İŞ şğüöç'
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
## Suppression
|
|
184
|
+
|
|
185
|
+
Some files contain broken characters on purpose — documentation about encoding bugs, test
|
|
186
|
+
fixtures, golden files. Two ways to say so:
|
|
187
|
+
|
|
188
|
+
```gitignore
|
|
189
|
+
# .mojiblameignore
|
|
190
|
+
fixtures/
|
|
191
|
+
*.snapshot
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
or put `mojiblame:ignore-file` anywhere in the file. The marker is matched against raw bytes,
|
|
195
|
+
so it still works in files that do not decode cleanly — which is exactly where you need it.
|
|
196
|
+
|
|
197
|
+
## Limitations
|
|
198
|
+
|
|
199
|
+
Stated plainly, because a forensics tool that oversells itself is worse than none:
|
|
200
|
+
|
|
201
|
+
- **Codec identification is statistical.** `latin-1` decodes every possible byte sequence, so a
|
|
202
|
+
clean round-trip proves the transcode is reversible, not that the codec was right. mojiblame
|
|
203
|
+
reports those as two separate facts and never merges them into one confident claim.
|
|
204
|
+
- **The codec *name* on a mojibake finding may be one of several.** Where two codepages map the
|
|
205
|
+
characters in a file identically, they are indistinguishable from the bytes — corruption caused
|
|
206
|
+
by cp1254 is reported as `latin-1 -> utf-8` when the text contains nothing that separates them.
|
|
207
|
+
The recovered text is the same either way, and it is the part that gets verified; only the label
|
|
208
|
+
is ambiguous.
|
|
209
|
+
- **Destroyed characters with no clean ancestor stay destroyed.** If a line was first written
|
|
210
|
+
already broken, no earlier version exists and mojiblame will tell you it recovered `0 / n`
|
|
211
|
+
rather than inventing text.
|
|
212
|
+
- **History walking assumes damage, once introduced, persists.** That assumption is verified,
|
|
213
|
+
and the search falls back to a linear scan when a file was broken, fixed, and broken again.
|
|
214
|
+
- Files over 5 MB are skipped by default.
|
|
215
|
+
|
|
216
|
+
## How it works
|
|
217
|
+
|
|
218
|
+
```
|
|
219
|
+
detect/facts.py BOM, line endings, strict UTF-8 decode - facts, no inference
|
|
220
|
+
detect/codecs.py D1: which legacy codec, and is the transcode reversible
|
|
221
|
+
detect/mojibake.py D2: repair plan, layer depth, and the inverse-plan proof
|
|
222
|
+
gitfx/repo.py git, always via bytes - never text=True, never the console codepage
|
|
223
|
+
gitfx/forensics.py binary-searches history for the commit that introduced the damage
|
|
224
|
+
gitfx/recover.py splices destroyed runs from the clean version, keeping later edits
|
|
225
|
+
fixing/journal.py gzipped originals so undo is byte-identical
|
|
226
|
+
hooks/ pre-commit entry point and the agent PreToolUse guard
|
|
227
|
+
```
|
|
228
|
+
|
|
229
|
+
One detail worth calling out: `gitfx/repo.py` never uses `subprocess(text=True)`. That decodes
|
|
230
|
+
with the console codepage — cp1254 on a Turkish Windows box — which is precisely the bug class
|
|
231
|
+
this tool hunts. A forensics tool that mangles its own evidence is worthless, so every git call
|
|
232
|
+
returns bytes and decoding is always an explicit, local decision.
|
|
233
|
+
|
|
234
|
+
## Why this exists
|
|
235
|
+
|
|
236
|
+
A memory system I run injected context into every session for months. It was writing `y���n�`
|
|
237
|
+
where it meant `yönetici`, and nothing ever failed — the sessions just quietly got a little
|
|
238
|
+
worse. The cause was a PowerShell 5.1 hook writing stdout in the console codepage.
|
|
239
|
+
|
|
240
|
+
Finding it took an afternoon. Knowing *when* it started took git archaeology by hand, and by
|
|
241
|
+
then the answer to "can any of this be recovered?" was pure guesswork. mojiblame is that
|
|
242
|
+
afternoon, made repeatable.
|
|
243
|
+
|
|
244
|
+
Its first real run, against that same 437-file vault, found four damaged files and named the
|
|
245
|
+
commit responsible for each.
|
|
246
|
+
|
|
247
|
+
## License
|
|
248
|
+
|
|
249
|
+
MIT
|
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
# mojiblame
|
|
2
|
+
|
|
3
|
+
<!-- mojiblame:ignore-file — this README intentionally contains damaged encoding examples. -->
|
|
4
|
+
|
|
5
|
+
**Git-aware encoding forensics.** Find the commit that corrupted your text, prove every fix
|
|
6
|
+
byte-exact, and recover characters other tools call unrecoverable.
|
|
7
|
+
|
|
8
|
+
[](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml)
|
|
9
|
+
[](https://pypi.org/project/mojiblame/)
|
|
10
|
+
[](https://pypi.org/project/mojiblame/)
|
|
11
|
+
[](LICENSE)
|
|
12
|
+
|
|
13
|
+

|
|
14
|
+
|
|
15
|
+
```console
|
|
16
|
+
$ mojiblame scan .
|
|
17
|
+
|
|
18
|
+
damage proof file detail
|
|
19
|
+
lossy unprovable docs/kurulum.md 19 character(s) destroyed (U+FFFD) - try: mojiblame blame
|
|
20
|
+
mojibake byte-exact src/tr.json latin-1 -> utf-8, 1 layer(s), reversal verified byte-exact
|
|
21
|
+
|
|
22
|
+
2 scanned · 2 damaged · 1 fixable now · 1 needs history (mojiblame blame <file>)
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
---
|
|
26
|
+
|
|
27
|
+
## The problem
|
|
28
|
+
|
|
29
|
+
AI coding agents read and write your files through shells and subprocesses. On Windows —
|
|
30
|
+
and anywhere the default codepage is not UTF-8 — that pipeline silently mangles non-ASCII
|
|
31
|
+
text. Nobody notices, because nothing fails: the agent writes `ö` instead of `ö`, the patch
|
|
32
|
+
context stops matching, a hook prints garbage, and the damage sits in the repo for weeks.
|
|
33
|
+
|
|
34
|
+
This is not hypothetical. It has been reported against
|
|
35
|
+
[Codex on Windows](https://github.com/openai/codex/issues/4131) — Turkish characters coming
|
|
36
|
+
back wrong from `Get-Content`/`Set-Content` — and again as
|
|
37
|
+
[mojibake breaking patch application](https://community.openai.com/t/codex-on-windows-utf-8-mojibake-and-patch-context-mismatch-issues/1384116),
|
|
38
|
+
where the agent can no longer edit a file because the bytes it read are not the bytes on disk.
|
|
39
|
+
|
|
40
|
+
Existing tools look at a file **as it is now** and **guess**. `ftfy` is an excellent library
|
|
41
|
+
for repairing a string, but it does not scan a repository, does not know your git history,
|
|
42
|
+
and cannot tell you whether its repair is correct or merely plausible.
|
|
43
|
+
|
|
44
|
+

|
|
45
|
+
|
|
46
|
+
## What mojiblame does differently
|
|
47
|
+
|
|
48
|
+
### 1. It separates three kinds of damage that other tools blur together
|
|
49
|
+
|
|
50
|
+
| | What happened | Fix | Standard of proof |
|
|
51
|
+
|---|---|---|---|
|
|
52
|
+
| **wrong-codec** | The bytes are not UTF-8 at all (cp1254, cp1251, latin-1…). The text is intact; only the codec is wrong. | transcode | **reversible** — re-encoding restores the original bytes. The codec itself is inferred, and we say so. |
|
|
53
|
+
| **mojibake** | The file *is* valid UTF-8, but the characters are wrong: UTF-8 was decoded as a legacy codepage and re-encoded. `ö` → `ö`. Stacks to several layers. | repair | **byte-exact** — see below |
|
|
54
|
+
| **lossy** | `U+FFFD` replacement characters. The bytes were destroyed at decode time. They are not in the file any more. | *none* | **unprovable from the file** — only history can help |
|
|
55
|
+
|
|
56
|
+
The distinction is not academic. A tool that treats the third case like the first will happily
|
|
57
|
+
"fix" a file and hand you plausible fiction.
|
|
58
|
+
|
|
59
|
+
### 2. Every fix is proven, or it is not applied
|
|
60
|
+
|
|
61
|
+
For mojibake we take the repair plan, **invert it**, and re-apply the inversion to the repaired
|
|
62
|
+
text. If that reproduces the bytes currently on disk exactly, the repair is not a guess: the
|
|
63
|
+
recovered text is a verified preimage of the file as it stands, not the most plausible-looking
|
|
64
|
+
candidate.
|
|
65
|
+
|
|
66
|
+
If the inversion does not reproduce them, mojiblame reports the finding and **refuses to write**:
|
|
67
|
+
|
|
68
|
+
```console
|
|
69
|
+
mojibake unprovable README.md looks like partially decoded UTF-8 mojibake,
|
|
70
|
+
but the reversal is not exact - not fixing
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
No proof, no write. That rule has no exceptions.
|
|
74
|
+
|
|
75
|
+
### 3. It recovers what the file alone cannot
|
|
76
|
+
|
|
77
|
+
`U+FFFD` means the bytes are gone, and no amount of cleverness gets them back out of the file.
|
|
78
|
+
But git kept every earlier version. `mojiblame blame` walks that history, finds the commit where
|
|
79
|
+
the damage appeared, works out which decode step caused it, and restores the destroyed
|
|
80
|
+
characters from the last clean version:
|
|
81
|
+
|
|
82
|
+
```console
|
|
83
|
+
$ mojiblame blame docs/kurulum.md
|
|
84
|
+
|
|
85
|
+
kurulum.md
|
|
86
|
+
|
|
87
|
+
19 character(s) destroyed (U+FFFD)
|
|
88
|
+
corruption entered 03bbd41 2026-09-02 chore: toplu yeniden bicimlendirme
|
|
89
|
+
last clean version c6455a1 2026-09-02 docs: kurulum notlari eklendi
|
|
90
|
+
transform cp1254 -> utf-8
|
|
91
|
+
recoverable 19 / 19 (proof: re-applying the transform reproduces this file exactly)
|
|
92
|
+
```
|
|
93
|
+
|
|
94
|
+
That last line is the point. We reconstruct the corruption, apply it to the recovered text, and
|
|
95
|
+
check it reproduces today's file byte for byte. If it does, the recovery is a deduction, not a
|
|
96
|
+
best guess — and mojiblame says which one it is either way.
|
|
97
|
+
|
|
98
|
+
**It splices, it does not revert.** `git checkout <old-sha> -- file` would throw away every
|
|
99
|
+
legitimate edit made since the damage. mojiblame keeps today's file and replaces only the runs
|
|
100
|
+
of destroyed characters.
|
|
101
|
+
|
|
102
|
+

|
|
103
|
+
|
|
104
|
+
## Install
|
|
105
|
+
|
|
106
|
+
```console
|
|
107
|
+
uv tool install mojiblame # or: pipx install mojiblame
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
## Use
|
|
111
|
+
|
|
112
|
+
```console
|
|
113
|
+
mojiblame scan [PATH] read-only report, with the standard of proof for each finding
|
|
114
|
+
mojiblame blame <FILE> which commit broke it, which step, what history can restore
|
|
115
|
+
mojiblame fix [PATH] apply only the provable fixes; --dry-run, --yes
|
|
116
|
+
mojiblame undo restore the originals from the last fix
|
|
117
|
+
mojiblame guard [PATH] exit non-zero if anything is corrupted (CI, hooks)
|
|
118
|
+
mojiblame init wire it into pre-commit, GitHub Actions, or your agent
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
`fix` journals the original bytes before it changes anything, so `undo` is byte-identical —
|
|
122
|
+
not "close enough". Inside a git repo, scanning honours `.gitignore`.
|
|
123
|
+
|
|
124
|
+
## Wire it in
|
|
125
|
+
|
|
126
|
+
**pre-commit** — the version most people should use, because it runs without anyone remembering it:
|
|
127
|
+
|
|
128
|
+
```yaml
|
|
129
|
+
repos:
|
|
130
|
+
- repo: https://github.com/Golemozan/mojiblame
|
|
131
|
+
rev: v0.1.0
|
|
132
|
+
hooks:
|
|
133
|
+
- id: mojiblame
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
**GitHub Actions** — `mojiblame init --github`, or:
|
|
137
|
+
|
|
138
|
+
```yaml
|
|
139
|
+
- run: pip install mojiblame
|
|
140
|
+
- run: mojiblame guard .
|
|
141
|
+
```
|
|
142
|
+
|
|
143
|
+
**Coding agents** — `mojiblame init --claude` installs a `PreToolUse` hook that inspects writes
|
|
144
|
+
*before they reach the disk* and blocks the ones that would introduce corruption. Cleaning up
|
|
145
|
+
afterwards is the losing half of this problem; the agent is told what it was about to corrupt
|
|
146
|
+
and can retry correctly:
|
|
147
|
+
|
|
148
|
+
```
|
|
149
|
+
mojiblame blocked this write to notes.md: mojibake (cp1254 -> utf-8, 1 layer(s)).
|
|
150
|
+
The text was decoded with the wrong codec somewhere upstream.
|
|
151
|
+
Intended text starts: 'yönetici İŞ şğüöç'
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
## Suppression
|
|
155
|
+
|
|
156
|
+
Some files contain broken characters on purpose — documentation about encoding bugs, test
|
|
157
|
+
fixtures, golden files. Two ways to say so:
|
|
158
|
+
|
|
159
|
+
```gitignore
|
|
160
|
+
# .mojiblameignore
|
|
161
|
+
fixtures/
|
|
162
|
+
*.snapshot
|
|
163
|
+
```
|
|
164
|
+
|
|
165
|
+
or put `mojiblame:ignore-file` anywhere in the file. The marker is matched against raw bytes,
|
|
166
|
+
so it still works in files that do not decode cleanly — which is exactly where you need it.
|
|
167
|
+
|
|
168
|
+
## Limitations
|
|
169
|
+
|
|
170
|
+
Stated plainly, because a forensics tool that oversells itself is worse than none:
|
|
171
|
+
|
|
172
|
+
- **Codec identification is statistical.** `latin-1` decodes every possible byte sequence, so a
|
|
173
|
+
clean round-trip proves the transcode is reversible, not that the codec was right. mojiblame
|
|
174
|
+
reports those as two separate facts and never merges them into one confident claim.
|
|
175
|
+
- **The codec *name* on a mojibake finding may be one of several.** Where two codepages map the
|
|
176
|
+
characters in a file identically, they are indistinguishable from the bytes — corruption caused
|
|
177
|
+
by cp1254 is reported as `latin-1 -> utf-8` when the text contains nothing that separates them.
|
|
178
|
+
The recovered text is the same either way, and it is the part that gets verified; only the label
|
|
179
|
+
is ambiguous.
|
|
180
|
+
- **Destroyed characters with no clean ancestor stay destroyed.** If a line was first written
|
|
181
|
+
already broken, no earlier version exists and mojiblame will tell you it recovered `0 / n`
|
|
182
|
+
rather than inventing text.
|
|
183
|
+
- **History walking assumes damage, once introduced, persists.** That assumption is verified,
|
|
184
|
+
and the search falls back to a linear scan when a file was broken, fixed, and broken again.
|
|
185
|
+
- Files over 5 MB are skipped by default.
|
|
186
|
+
|
|
187
|
+
## How it works
|
|
188
|
+
|
|
189
|
+
```
|
|
190
|
+
detect/facts.py BOM, line endings, strict UTF-8 decode - facts, no inference
|
|
191
|
+
detect/codecs.py D1: which legacy codec, and is the transcode reversible
|
|
192
|
+
detect/mojibake.py D2: repair plan, layer depth, and the inverse-plan proof
|
|
193
|
+
gitfx/repo.py git, always via bytes - never text=True, never the console codepage
|
|
194
|
+
gitfx/forensics.py binary-searches history for the commit that introduced the damage
|
|
195
|
+
gitfx/recover.py splices destroyed runs from the clean version, keeping later edits
|
|
196
|
+
fixing/journal.py gzipped originals so undo is byte-identical
|
|
197
|
+
hooks/ pre-commit entry point and the agent PreToolUse guard
|
|
198
|
+
```
|
|
199
|
+
|
|
200
|
+
One detail worth calling out: `gitfx/repo.py` never uses `subprocess(text=True)`. That decodes
|
|
201
|
+
with the console codepage — cp1254 on a Turkish Windows box — which is precisely the bug class
|
|
202
|
+
this tool hunts. A forensics tool that mangles its own evidence is worthless, so every git call
|
|
203
|
+
returns bytes and decoding is always an explicit, local decision.
|
|
204
|
+
|
|
205
|
+
## Why this exists
|
|
206
|
+
|
|
207
|
+
A memory system I run injected context into every session for months. It was writing `y���n�`
|
|
208
|
+
where it meant `yönetici`, and nothing ever failed — the sessions just quietly got a little
|
|
209
|
+
worse. The cause was a PowerShell 5.1 hook writing stdout in the console codepage.
|
|
210
|
+
|
|
211
|
+
Finding it took an afternoon. Knowing *when* it started took git archaeology by hand, and by
|
|
212
|
+
then the answer to "can any of this be recovered?" was pure guesswork. mojiblame is that
|
|
213
|
+
afternoon, made repeatable.
|
|
214
|
+
|
|
215
|
+
Its first real run, against that same 437-file vault, found four damaged files and named the
|
|
216
|
+
commit responsible for each.
|
|
217
|
+
|
|
218
|
+
## License
|
|
219
|
+
|
|
220
|
+
MIT
|
|
Binary file
|
|
Binary file
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" width="1400" height="430" viewBox="0 0 1400 430" role="img" aria-labelledby="title desc">
|
|
2
|
+
<!-- mojiblame:ignore-file — the final node intentionally shows a mojibake sample. -->
|
|
3
|
+
<title id="title">How silent encoding corruption enters a repository</title>
|
|
4
|
+
<desc id="desc">A UTF-8 file is read by an agent or tool through a shell, decoded with the wrong codepage, and written back as mojibake. Every step reports success.</desc>
|
|
5
|
+
<defs>
|
|
6
|
+
<linearGradient id="panel" x1="0" y1="0" x2="1" y2="1">
|
|
7
|
+
<stop offset="0" stop-color="#10171c"/>
|
|
8
|
+
<stop offset="1" stop-color="#080c0f"/>
|
|
9
|
+
</linearGradient>
|
|
10
|
+
<filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
|
|
11
|
+
<feDropShadow dx="0" dy="8" stdDeviation="10" flood-color="#000" flood-opacity=".28"/>
|
|
12
|
+
</filter>
|
|
13
|
+
<marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
|
|
14
|
+
<path d="M 0 0 L 10 5 L 0 10 z" fill="#93a1aa"/>
|
|
15
|
+
</marker>
|
|
16
|
+
<style>
|
|
17
|
+
.label { font: 700 14px ui-monospace, SFMono-Regular, Consolas, monospace; letter-spacing: .12em; fill: #ffab00; }
|
|
18
|
+
.title { font: 700 21px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #edf1f3; }
|
|
19
|
+
.body { font: 400 16px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #aeb9bf; }
|
|
20
|
+
.code { font: 600 18px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #edf1f3; }
|
|
21
|
+
.arrow-label { font: 600 13px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #7f8c94; letter-spacing: .08em; }
|
|
22
|
+
</style>
|
|
23
|
+
</defs>
|
|
24
|
+
|
|
25
|
+
<rect x="1" y="1" width="1398" height="428" rx="18" fill="url(#panel)" stroke="#34424a" stroke-width="2"/>
|
|
26
|
+
<text x="54" y="51" class="label">THE SILENT CORRUPTION PATH</text>
|
|
27
|
+
<text x="1346" y="51" text-anchor="end" class="body">deterministic · no inference</text>
|
|
28
|
+
|
|
29
|
+
<g filter="url(#shadow)">
|
|
30
|
+
<g transform="translate(54 92)">
|
|
31
|
+
<rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
|
|
32
|
+
<text x="24" y="36" class="label">01 · FILE</text>
|
|
33
|
+
<text x="24" y="76" class="title">UTF-8 on disk</text>
|
|
34
|
+
<rect x="24" y="100" width="212" height="48" rx="7" fill="#080c0f" stroke="#2c3940"/>
|
|
35
|
+
<text x="42" y="131" class="code">yönetici</text>
|
|
36
|
+
</g>
|
|
37
|
+
|
|
38
|
+
<g transform="translate(397 92)">
|
|
39
|
+
<rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
|
|
40
|
+
<text x="24" y="36" class="label">02 · READ</text>
|
|
41
|
+
<text x="24" y="76" class="title">agent / tool</text>
|
|
42
|
+
<text x="24" y="111" class="body">shell</text>
|
|
43
|
+
<text x="24" y="140" class="body">subprocess stdout</text>
|
|
44
|
+
</g>
|
|
45
|
+
|
|
46
|
+
<g transform="translate(740 92)">
|
|
47
|
+
<rect width="260" height="176" rx="13" fill="#1c1710" stroke="#ffab00" stroke-width="2"/>
|
|
48
|
+
<text x="24" y="36" class="label">03 · DECODE</text>
|
|
49
|
+
<text x="24" y="76" class="title">wrong codepage</text>
|
|
50
|
+
<text x="24" y="113" class="code">cp1254</text>
|
|
51
|
+
<text x="24" y="143" class="body">bytes ≠ characters</text>
|
|
52
|
+
</g>
|
|
53
|
+
|
|
54
|
+
<g transform="translate(1083 92)">
|
|
55
|
+
<rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
|
|
56
|
+
<text x="24" y="36" class="label">04 · WRITE BACK</text>
|
|
57
|
+
<text x="24" y="76" class="title">valid UTF-8</text>
|
|
58
|
+
<rect x="24" y="100" width="212" height="48" rx="7" fill="#080c0f" stroke="#ffab00"/>
|
|
59
|
+
<text x="42" y="131" class="code" fill="#ffbf3f">yönetici</text>
|
|
60
|
+
</g>
|
|
61
|
+
</g>
|
|
62
|
+
|
|
63
|
+
<g fill="none" stroke="#93a1aa" stroke-width="2" marker-end="url(#arrow)">
|
|
64
|
+
<path d="M 320 180 H 381"/>
|
|
65
|
+
<path d="M 663 180 H 724"/>
|
|
66
|
+
<path d="M 1006 180 H 1067"/>
|
|
67
|
+
</g>
|
|
68
|
+
<text x="350" y="164" text-anchor="middle" class="arrow-label">READ</text>
|
|
69
|
+
<text x="693" y="164" text-anchor="middle" class="arrow-label">TEXT</text>
|
|
70
|
+
<text x="1036" y="164" text-anchor="middle" class="arrow-label">WRITE</text>
|
|
71
|
+
|
|
72
|
+
<rect x="54" y="309" width="1289" height="76" rx="12" fill="#18140c" stroke="#ffab00" stroke-width="2"/>
|
|
73
|
+
<circle cx="91" cy="347" r="17" fill="#ffab00"/>
|
|
74
|
+
<path d="M 91 335 V 351 M 91 361 V 362" stroke="#111" stroke-width="4" stroke-linecap="round"/>
|
|
75
|
+
<text x="126" y="341" class="title">NO STEP THROWS AN ERROR.</text>
|
|
76
|
+
<text x="126" y="369" class="body">Read succeeds. Decode succeeds. Write succeeds. The silence is the bug.</text>
|
|
77
|
+
</svg>
|