mojiblame 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (41) hide show
  1. mojiblame-0.1.0/.github/workflows/publish.yml +26 -0
  2. mojiblame-0.1.0/.github/workflows/test.yml +38 -0
  3. mojiblame-0.1.0/.gitignore +15 -0
  4. mojiblame-0.1.0/.mojiblameignore +4 -0
  5. mojiblame-0.1.0/.pre-commit-hooks.yaml +8 -0
  6. mojiblame-0.1.0/LICENSE +21 -0
  7. mojiblame-0.1.0/PKG-INFO +249 -0
  8. mojiblame-0.1.0/README.md +220 -0
  9. mojiblame-0.1.0/docs/assets/blame-demo.gif +0 -0
  10. mojiblame-0.1.0/docs/assets/hero.png +0 -0
  11. mojiblame-0.1.0/docs/assets/pipeline.svg +77 -0
  12. mojiblame-0.1.0/pyproject.toml +50 -0
  13. mojiblame-0.1.0/src/mojiblame/__init__.py +3 -0
  14. mojiblame-0.1.0/src/mojiblame/cli.py +259 -0
  15. mojiblame-0.1.0/src/mojiblame/detect/__init__.py +0 -0
  16. mojiblame-0.1.0/src/mojiblame/detect/codecs.py +66 -0
  17. mojiblame-0.1.0/src/mojiblame/detect/facts.py +104 -0
  18. mojiblame-0.1.0/src/mojiblame/detect/mojibake.py +114 -0
  19. mojiblame-0.1.0/src/mojiblame/fixing/__init__.py +0 -0
  20. mojiblame-0.1.0/src/mojiblame/fixing/apply.py +51 -0
  21. mojiblame-0.1.0/src/mojiblame/fixing/journal.py +79 -0
  22. mojiblame-0.1.0/src/mojiblame/gitfx/__init__.py +0 -0
  23. mojiblame-0.1.0/src/mojiblame/gitfx/forensics.py +197 -0
  24. mojiblame-0.1.0/src/mojiblame/gitfx/recover.py +85 -0
  25. mojiblame-0.1.0/src/mojiblame/gitfx/repo.py +125 -0
  26. mojiblame-0.1.0/src/mojiblame/hooks/__init__.py +0 -0
  27. mojiblame-0.1.0/src/mojiblame/hooks/agent.py +93 -0
  28. mojiblame-0.1.0/src/mojiblame/hooks/precommit.py +39 -0
  29. mojiblame-0.1.0/src/mojiblame/ignore.py +63 -0
  30. mojiblame-0.1.0/src/mojiblame/model.py +127 -0
  31. mojiblame-0.1.0/src/mojiblame/report/__init__.py +0 -0
  32. mojiblame-0.1.0/src/mojiblame/report/render.py +158 -0
  33. mojiblame-0.1.0/src/mojiblame/scanner.py +178 -0
  34. mojiblame-0.1.0/tests/__init__.py +0 -0
  35. mojiblame-0.1.0/tests/conftest.py +77 -0
  36. mojiblame-0.1.0/tests/test_detect.py +119 -0
  37. mojiblame-0.1.0/tests/test_forensics.py +141 -0
  38. mojiblame-0.1.0/tests/test_hooks_and_cli.py +178 -0
  39. mojiblame-0.1.0/tests/test_ignore.py +61 -0
  40. mojiblame-0.1.0/tests/test_scanner_and_fix.py +100 -0
  41. mojiblame-0.1.0/uv.lock +419 -0
@@ -0,0 +1,26 @@
1
+ name: publish
2
+
3
+ on:
4
+ push:
5
+ tags: ["v*"]
6
+
7
+ jobs:
8
+ publish:
9
+ runs-on: ubuntu-latest
10
+ environment: pypi
11
+ permissions:
12
+ # Trusted publishing: PyPI verifies this workflow's identity directly, so
13
+ # there is no long-lived API token sitting in repository secrets.
14
+ id-token: write
15
+
16
+ steps:
17
+ - uses: actions/checkout@v4
18
+ - uses: astral-sh/setup-uv@v5
19
+ with:
20
+ python-version: "3.12"
21
+
22
+ - run: uv sync --all-extras --dev
23
+ - run: uv run python -m pytest -q
24
+ - run: uv build
25
+ - name: Publish to PyPI
26
+ run: uv publish --trusted-publishing always
@@ -0,0 +1,38 @@
1
+ name: test
2
+
3
+ on:
4
+ push:
5
+ branches: [master, main]
6
+ pull_request:
7
+
8
+ jobs:
9
+ test:
10
+ # Windows is not an afterthought here - it is where most of this damage is
11
+ # created, so a green Windows job is the point rather than a bonus.
12
+ runs-on: ${{ matrix.os }}
13
+ strategy:
14
+ fail-fast: false
15
+ matrix:
16
+ os: [ubuntu-latest, windows-latest, macos-latest]
17
+ python-version: ["3.10", "3.11", "3.12", "3.13"]
18
+
19
+ steps:
20
+ - uses: actions/checkout@v4
21
+
22
+ - name: Install uv
23
+ uses: astral-sh/setup-uv@v5
24
+ with:
25
+ python-version: ${{ matrix.python-version }}
26
+
27
+ - name: Configure git for the synthetic-repo tests
28
+ run: |
29
+ git config --global user.email "ci@example.invalid"
30
+ git config --global user.name "ci"
31
+ git config --global init.defaultBranch main
32
+
33
+ - run: uv sync --all-extras --dev
34
+
35
+ - run: uv run python -m pytest -q
36
+
37
+ - name: Dogfood - the tool must find nothing in its own source
38
+ run: uv run mojiblame scan .
@@ -0,0 +1,15 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ build/
6
+ *.egg-info/
7
+ .pytest_cache/
8
+ .ruff_cache/
9
+ .mypy_cache/
10
+
11
+ # mojiblame's own undo journals
12
+ .mojiblame/
13
+
14
+ # Ic devir notu - repoda yeri yok
15
+ docs/BRIEF-visuals.md
@@ -0,0 +1,4 @@
1
+ # Test fixtures and the visuals brief carry broken characters on purpose:
2
+ # they are the samples this tool is built to recognise.
3
+ tests/
4
+ docs/BRIEF-visuals.md
@@ -0,0 +1,8 @@
1
+ - id: mojiblame
2
+ name: mojiblame (encoding forensics)
3
+ description: >-
4
+ Refuses commits containing mojibake, non-UTF-8 files, or characters that were
5
+ destroyed by a bad decode.
6
+ entry: mojiblame-precommit
7
+ language: python
8
+ types: [text]
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Ozan
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,249 @@
1
+ Metadata-Version: 2.5
2
+ Name: mojiblame
3
+ Version: 0.1.0
4
+ Summary: Git-aware encoding forensics: find which commit corrupted your text, prove every fix byte-exact, recover what other tools call unrecoverable.
5
+ Project-URL: Homepage, https://github.com/Golemozan/mojiblame
6
+ Project-URL: Repository, https://github.com/Golemozan/mojiblame
7
+ Project-URL: Issues, https://github.com/Golemozan/mojiblame/issues
8
+ Author: Ozan
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: cp1252,encoding,forensics,git,mojibake,pre-commit,utf-8,windows
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Environment :: Console
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: License :: OSI Approved :: MIT License
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Software Development :: Quality Assurance
22
+ Classifier: Topic :: Text Processing :: General
23
+ Requires-Python: >=3.10
24
+ Requires-Dist: charset-normalizer>=3.3
25
+ Requires-Dist: ftfy>=6.2
26
+ Requires-Dist: rich>=13.7
27
+ Requires-Dist: typer>=0.12
28
+ Description-Content-Type: text/markdown
29
+
30
+ # mojiblame
31
+
32
+ <!-- mojiblame:ignore-file — this README intentionally contains damaged encoding examples. -->
33
+
34
+ **Git-aware encoding forensics.** Find the commit that corrupted your text, prove every fix
35
+ byte-exact, and recover characters other tools call unrecoverable.
36
+
37
+ [![test](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml/badge.svg)](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml)
38
+ [![PyPI](https://img.shields.io/pypi/v/mojiblame)](https://pypi.org/project/mojiblame/)
39
+ [![Python](https://img.shields.io/pypi/pyversions/mojiblame)](https://pypi.org/project/mojiblame/)
40
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
41
+
42
+ ![mojiblame — git-aware encoding forensics](docs/assets/hero.png)
43
+
44
+ ```console
45
+ $ mojiblame scan .
46
+
47
+ damage proof file detail
48
+ lossy unprovable docs/kurulum.md 19 character(s) destroyed (U+FFFD) - try: mojiblame blame
49
+ mojibake byte-exact src/tr.json latin-1 -> utf-8, 1 layer(s), reversal verified byte-exact
50
+
51
+ 2 scanned · 2 damaged · 1 fixable now · 1 needs history (mojiblame blame <file>)
52
+ ```
53
+
54
+ ---
55
+
56
+ ## The problem
57
+
58
+ AI coding agents read and write your files through shells and subprocesses. On Windows —
59
+ and anywhere the default codepage is not UTF-8 — that pipeline silently mangles non-ASCII
60
+ text. Nobody notices, because nothing fails: the agent writes `ö` instead of `ö`, the patch
61
+ context stops matching, a hook prints garbage, and the damage sits in the repo for weeks.
62
+
63
+ This is not hypothetical. It has been reported against
64
+ [Codex on Windows](https://github.com/openai/codex/issues/4131) — Turkish characters coming
65
+ back wrong from `Get-Content`/`Set-Content` — and again as
66
+ [mojibake breaking patch application](https://community.openai.com/t/codex-on-windows-utf-8-mojibake-and-patch-context-mismatch-issues/1384116),
67
+ where the agent can no longer edit a file because the bytes it read are not the bytes on disk.
68
+
69
+ Existing tools look at a file **as it is now** and **guess**. `ftfy` is an excellent library
70
+ for repairing a string, but it does not scan a repository, does not know your git history,
71
+ and cannot tell you whether its repair is correct or merely plausible.
72
+
73
+ ![How silent codepage corruption enters a repository](docs/assets/pipeline.svg)
74
+
75
+ ## What mojiblame does differently
76
+
77
+ ### 1. It separates three kinds of damage that other tools blur together
78
+
79
+ | | What happened | Fix | Standard of proof |
80
+ |---|---|---|---|
81
+ | **wrong-codec** | The bytes are not UTF-8 at all (cp1254, cp1251, latin-1…). The text is intact; only the codec is wrong. | transcode | **reversible** — re-encoding restores the original bytes. The codec itself is inferred, and we say so. |
82
+ | **mojibake** | The file *is* valid UTF-8, but the characters are wrong: UTF-8 was decoded as a legacy codepage and re-encoded. `ö` → `ö`. Stacks to several layers. | repair | **byte-exact** — see below |
83
+ | **lossy** | `U+FFFD` replacement characters. The bytes were destroyed at decode time. They are not in the file any more. | *none* | **unprovable from the file** — only history can help |
84
+
85
+ The distinction is not academic. A tool that treats the third case like the first will happily
86
+ "fix" a file and hand you plausible fiction.
87
+
88
+ ### 2. Every fix is proven, or it is not applied
89
+
90
+ For mojibake we take the repair plan, **invert it**, and re-apply the inversion to the repaired
91
+ text. If that reproduces the bytes currently on disk exactly, the repair is not a guess: the
92
+ recovered text is a verified preimage of the file as it stands, not the most plausible-looking
93
+ candidate.
94
+
95
+ If the inversion does not reproduce them, mojiblame reports the finding and **refuses to write**:
96
+
97
+ ```console
98
+ mojibake unprovable README.md looks like partially decoded UTF-8 mojibake,
99
+ but the reversal is not exact - not fixing
100
+ ```
101
+
102
+ No proof, no write. That rule has no exceptions.
103
+
104
+ ### 3. It recovers what the file alone cannot
105
+
106
+ `U+FFFD` means the bytes are gone, and no amount of cleverness gets them back out of the file.
107
+ But git kept every earlier version. `mojiblame blame` walks that history, finds the commit where
108
+ the damage appeared, works out which decode step caused it, and restores the destroyed
109
+ characters from the last clean version:
110
+
111
+ ```console
112
+ $ mojiblame blame docs/kurulum.md
113
+
114
+ kurulum.md
115
+
116
+ 19 character(s) destroyed (U+FFFD)
117
+ corruption entered 03bbd41 2026-09-02 chore: toplu yeniden bicimlendirme
118
+ last clean version c6455a1 2026-09-02 docs: kurulum notlari eklendi
119
+ transform cp1254 -> utf-8
120
+ recoverable 19 / 19 (proof: re-applying the transform reproduces this file exactly)
121
+ ```
122
+
123
+ That last line is the point. We reconstruct the corruption, apply it to the recovered text, and
124
+ check it reproduces today's file byte for byte. If it does, the recovery is a deduction, not a
125
+ best guess — and mojiblame says which one it is either way.
126
+
127
+ **It splices, it does not revert.** `git checkout <old-sha> -- file` would throw away every
128
+ legitimate edit made since the damage. mojiblame keeps today's file and replaces only the runs
129
+ of destroyed characters.
130
+
131
+ ![mojiblame scan, blame, and fix terminal demo](docs/assets/blame-demo.gif)
132
+
133
+ ## Install
134
+
135
+ ```console
136
+ uv tool install mojiblame # or: pipx install mojiblame
137
+ ```
138
+
139
+ ## Use
140
+
141
+ ```console
142
+ mojiblame scan [PATH] read-only report, with the standard of proof for each finding
143
+ mojiblame blame <FILE> which commit broke it, which step, what history can restore
144
+ mojiblame fix [PATH] apply only the provable fixes; --dry-run, --yes
145
+ mojiblame undo restore the originals from the last fix
146
+ mojiblame guard [PATH] exit non-zero if anything is corrupted (CI, hooks)
147
+ mojiblame init wire it into pre-commit, GitHub Actions, or your agent
148
+ ```
149
+
150
+ `fix` journals the original bytes before it changes anything, so `undo` is byte-identical —
151
+ not "close enough". Inside a git repo, scanning honours `.gitignore`.
152
+
153
+ ## Wire it in
154
+
155
+ **pre-commit** — the version most people should use, because it runs without anyone remembering it:
156
+
157
+ ```yaml
158
+ repos:
159
+ - repo: https://github.com/Golemozan/mojiblame
160
+ rev: v0.1.0
161
+ hooks:
162
+ - id: mojiblame
163
+ ```
164
+
165
+ **GitHub Actions** — `mojiblame init --github`, or:
166
+
167
+ ```yaml
168
+ - run: pip install mojiblame
169
+ - run: mojiblame guard .
170
+ ```
171
+
172
+ **Coding agents** — `mojiblame init --claude` installs a `PreToolUse` hook that inspects writes
173
+ *before they reach the disk* and blocks the ones that would introduce corruption. Cleaning up
174
+ afterwards is the losing half of this problem; the agent is told what it was about to corrupt
175
+ and can retry correctly:
176
+
177
+ ```
178
+ mojiblame blocked this write to notes.md: mojibake (cp1254 -> utf-8, 1 layer(s)).
179
+ The text was decoded with the wrong codec somewhere upstream.
180
+ Intended text starts: 'yönetici İŞ şğüöç'
181
+ ```
182
+
183
+ ## Suppression
184
+
185
+ Some files contain broken characters on purpose — documentation about encoding bugs, test
186
+ fixtures, golden files. Two ways to say so:
187
+
188
+ ```gitignore
189
+ # .mojiblameignore
190
+ fixtures/
191
+ *.snapshot
192
+ ```
193
+
194
+ or put `mojiblame:ignore-file` anywhere in the file. The marker is matched against raw bytes,
195
+ so it still works in files that do not decode cleanly — which is exactly where you need it.
196
+
197
+ ## Limitations
198
+
199
+ Stated plainly, because a forensics tool that oversells itself is worse than none:
200
+
201
+ - **Codec identification is statistical.** `latin-1` decodes every possible byte sequence, so a
202
+ clean round-trip proves the transcode is reversible, not that the codec was right. mojiblame
203
+ reports those as two separate facts and never merges them into one confident claim.
204
+ - **The codec *name* on a mojibake finding may be one of several.** Where two codepages map the
205
+ characters in a file identically, they are indistinguishable from the bytes — corruption caused
206
+ by cp1254 is reported as `latin-1 -> utf-8` when the text contains nothing that separates them.
207
+ The recovered text is the same either way, and it is the part that gets verified; only the label
208
+ is ambiguous.
209
+ - **Destroyed characters with no clean ancestor stay destroyed.** If a line was first written
210
+ already broken, no earlier version exists and mojiblame will tell you it recovered `0 / n`
211
+ rather than inventing text.
212
+ - **History walking assumes damage, once introduced, persists.** That assumption is verified,
213
+ and the search falls back to a linear scan when a file was broken, fixed, and broken again.
214
+ - Files over 5 MB are skipped by default.
215
+
216
+ ## How it works
217
+
218
+ ```
219
+ detect/facts.py BOM, line endings, strict UTF-8 decode - facts, no inference
220
+ detect/codecs.py D1: which legacy codec, and is the transcode reversible
221
+ detect/mojibake.py D2: repair plan, layer depth, and the inverse-plan proof
222
+ gitfx/repo.py git, always via bytes - never text=True, never the console codepage
223
+ gitfx/forensics.py binary-searches history for the commit that introduced the damage
224
+ gitfx/recover.py splices destroyed runs from the clean version, keeping later edits
225
+ fixing/journal.py gzipped originals so undo is byte-identical
226
+ hooks/ pre-commit entry point and the agent PreToolUse guard
227
+ ```
228
+
229
+ One detail worth calling out: `gitfx/repo.py` never uses `subprocess(text=True)`. That decodes
230
+ with the console codepage — cp1254 on a Turkish Windows box — which is precisely the bug class
231
+ this tool hunts. A forensics tool that mangles its own evidence is worthless, so every git call
232
+ returns bytes and decoding is always an explicit, local decision.
233
+
234
+ ## Why this exists
235
+
236
+ A memory system I run injected context into every session for months. It was writing `y���n�`
237
+ where it meant `yönetici`, and nothing ever failed — the sessions just quietly got a little
238
+ worse. The cause was a PowerShell 5.1 hook writing stdout in the console codepage.
239
+
240
+ Finding it took an afternoon. Knowing *when* it started took git archaeology by hand, and by
241
+ then the answer to "can any of this be recovered?" was pure guesswork. mojiblame is that
242
+ afternoon, made repeatable.
243
+
244
+ Its first real run, against that same 437-file vault, found four damaged files and named the
245
+ commit responsible for each.
246
+
247
+ ## License
248
+
249
+ MIT
@@ -0,0 +1,220 @@
1
+ # mojiblame
2
+
3
+ <!-- mojiblame:ignore-file — this README intentionally contains damaged encoding examples. -->
4
+
5
+ **Git-aware encoding forensics.** Find the commit that corrupted your text, prove every fix
6
+ byte-exact, and recover characters other tools call unrecoverable.
7
+
8
+ [![test](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml/badge.svg)](https://github.com/Golemozan/mojiblame/actions/workflows/test.yml)
9
+ [![PyPI](https://img.shields.io/pypi/v/mojiblame)](https://pypi.org/project/mojiblame/)
10
+ [![Python](https://img.shields.io/pypi/pyversions/mojiblame)](https://pypi.org/project/mojiblame/)
11
+ [![License: MIT](https://img.shields.io/badge/license-MIT-green)](LICENSE)
12
+
13
+ ![mojiblame — git-aware encoding forensics](docs/assets/hero.png)
14
+
15
+ ```console
16
+ $ mojiblame scan .
17
+
18
+ damage proof file detail
19
+ lossy unprovable docs/kurulum.md 19 character(s) destroyed (U+FFFD) - try: mojiblame blame
20
+ mojibake byte-exact src/tr.json latin-1 -> utf-8, 1 layer(s), reversal verified byte-exact
21
+
22
+ 2 scanned · 2 damaged · 1 fixable now · 1 needs history (mojiblame blame <file>)
23
+ ```
24
+
25
+ ---
26
+
27
+ ## The problem
28
+
29
+ AI coding agents read and write your files through shells and subprocesses. On Windows —
30
+ and anywhere the default codepage is not UTF-8 — that pipeline silently mangles non-ASCII
31
+ text. Nobody notices, because nothing fails: the agent writes `ö` instead of `ö`, the patch
32
+ context stops matching, a hook prints garbage, and the damage sits in the repo for weeks.
33
+
34
+ This is not hypothetical. It has been reported against
35
+ [Codex on Windows](https://github.com/openai/codex/issues/4131) — Turkish characters coming
36
+ back wrong from `Get-Content`/`Set-Content` — and again as
37
+ [mojibake breaking patch application](https://community.openai.com/t/codex-on-windows-utf-8-mojibake-and-patch-context-mismatch-issues/1384116),
38
+ where the agent can no longer edit a file because the bytes it read are not the bytes on disk.
39
+
40
+ Existing tools look at a file **as it is now** and **guess**. `ftfy` is an excellent library
41
+ for repairing a string, but it does not scan a repository, does not know your git history,
42
+ and cannot tell you whether its repair is correct or merely plausible.
43
+
44
+ ![How silent codepage corruption enters a repository](docs/assets/pipeline.svg)
45
+
46
+ ## What mojiblame does differently
47
+
48
+ ### 1. It separates three kinds of damage that other tools blur together
49
+
50
+ | | What happened | Fix | Standard of proof |
51
+ |---|---|---|---|
52
+ | **wrong-codec** | The bytes are not UTF-8 at all (cp1254, cp1251, latin-1…). The text is intact; only the codec is wrong. | transcode | **reversible** — re-encoding restores the original bytes. The codec itself is inferred, and we say so. |
53
+ | **mojibake** | The file *is* valid UTF-8, but the characters are wrong: UTF-8 was decoded as a legacy codepage and re-encoded. `ö` → `ö`. Stacks to several layers. | repair | **byte-exact** — see below |
54
+ | **lossy** | `U+FFFD` replacement characters. The bytes were destroyed at decode time. They are not in the file any more. | *none* | **unprovable from the file** — only history can help |
55
+
56
+ The distinction is not academic. A tool that treats the third case like the first will happily
57
+ "fix" a file and hand you plausible fiction.
58
+
59
+ ### 2. Every fix is proven, or it is not applied
60
+
61
+ For mojibake we take the repair plan, **invert it**, and re-apply the inversion to the repaired
62
+ text. If that reproduces the bytes currently on disk exactly, the repair is not a guess: the
63
+ recovered text is a verified preimage of the file as it stands, not the most plausible-looking
64
+ candidate.
65
+
66
+ If the inversion does not reproduce them, mojiblame reports the finding and **refuses to write**:
67
+
68
+ ```console
69
+ mojibake unprovable README.md looks like partially decoded UTF-8 mojibake,
70
+ but the reversal is not exact - not fixing
71
+ ```
72
+
73
+ No proof, no write. That rule has no exceptions.
74
+
75
+ ### 3. It recovers what the file alone cannot
76
+
77
+ `U+FFFD` means the bytes are gone, and no amount of cleverness gets them back out of the file.
78
+ But git kept every earlier version. `mojiblame blame` walks that history, finds the commit where
79
+ the damage appeared, works out which decode step caused it, and restores the destroyed
80
+ characters from the last clean version:
81
+
82
+ ```console
83
+ $ mojiblame blame docs/kurulum.md
84
+
85
+ kurulum.md
86
+
87
+ 19 character(s) destroyed (U+FFFD)
88
+ corruption entered 03bbd41 2026-09-02 chore: toplu yeniden bicimlendirme
89
+ last clean version c6455a1 2026-09-02 docs: kurulum notlari eklendi
90
+ transform cp1254 -> utf-8
91
+ recoverable 19 / 19 (proof: re-applying the transform reproduces this file exactly)
92
+ ```
93
+
94
+ That last line is the point. We reconstruct the corruption, apply it to the recovered text, and
95
+ check it reproduces today's file byte for byte. If it does, the recovery is a deduction, not a
96
+ best guess — and mojiblame says which one it is either way.
97
+
98
+ **It splices, it does not revert.** `git checkout <old-sha> -- file` would throw away every
99
+ legitimate edit made since the damage. mojiblame keeps today's file and replaces only the runs
100
+ of destroyed characters.
101
+
102
+ ![mojiblame scan, blame, and fix terminal demo](docs/assets/blame-demo.gif)
103
+
104
+ ## Install
105
+
106
+ ```console
107
+ uv tool install mojiblame # or: pipx install mojiblame
108
+ ```
109
+
110
+ ## Use
111
+
112
+ ```console
113
+ mojiblame scan [PATH] read-only report, with the standard of proof for each finding
114
+ mojiblame blame <FILE> which commit broke it, which step, what history can restore
115
+ mojiblame fix [PATH] apply only the provable fixes; --dry-run, --yes
116
+ mojiblame undo restore the originals from the last fix
117
+ mojiblame guard [PATH] exit non-zero if anything is corrupted (CI, hooks)
118
+ mojiblame init wire it into pre-commit, GitHub Actions, or your agent
119
+ ```
120
+
121
+ `fix` journals the original bytes before it changes anything, so `undo` is byte-identical —
122
+ not "close enough". Inside a git repo, scanning honours `.gitignore`.
123
+
124
+ ## Wire it in
125
+
126
+ **pre-commit** — the version most people should use, because it runs without anyone remembering it:
127
+
128
+ ```yaml
129
+ repos:
130
+ - repo: https://github.com/Golemozan/mojiblame
131
+ rev: v0.1.0
132
+ hooks:
133
+ - id: mojiblame
134
+ ```
135
+
136
+ **GitHub Actions** — `mojiblame init --github`, or:
137
+
138
+ ```yaml
139
+ - run: pip install mojiblame
140
+ - run: mojiblame guard .
141
+ ```
142
+
143
+ **Coding agents** — `mojiblame init --claude` installs a `PreToolUse` hook that inspects writes
144
+ *before they reach the disk* and blocks the ones that would introduce corruption. Cleaning up
145
+ afterwards is the losing half of this problem; the agent is told what it was about to corrupt
146
+ and can retry correctly:
147
+
148
+ ```
149
+ mojiblame blocked this write to notes.md: mojibake (cp1254 -> utf-8, 1 layer(s)).
150
+ The text was decoded with the wrong codec somewhere upstream.
151
+ Intended text starts: 'yönetici İŞ şğüöç'
152
+ ```
153
+
154
+ ## Suppression
155
+
156
+ Some files contain broken characters on purpose — documentation about encoding bugs, test
157
+ fixtures, golden files. Two ways to say so:
158
+
159
+ ```gitignore
160
+ # .mojiblameignore
161
+ fixtures/
162
+ *.snapshot
163
+ ```
164
+
165
+ or put `mojiblame:ignore-file` anywhere in the file. The marker is matched against raw bytes,
166
+ so it still works in files that do not decode cleanly — which is exactly where you need it.
167
+
168
+ ## Limitations
169
+
170
+ Stated plainly, because a forensics tool that oversells itself is worse than none:
171
+
172
+ - **Codec identification is statistical.** `latin-1` decodes every possible byte sequence, so a
173
+ clean round-trip proves the transcode is reversible, not that the codec was right. mojiblame
174
+ reports those as two separate facts and never merges them into one confident claim.
175
+ - **The codec *name* on a mojibake finding may be one of several.** Where two codepages map the
176
+ characters in a file identically, they are indistinguishable from the bytes — corruption caused
177
+ by cp1254 is reported as `latin-1 -> utf-8` when the text contains nothing that separates them.
178
+ The recovered text is the same either way, and it is the part that gets verified; only the label
179
+ is ambiguous.
180
+ - **Destroyed characters with no clean ancestor stay destroyed.** If a line was first written
181
+ already broken, no earlier version exists and mojiblame will tell you it recovered `0 / n`
182
+ rather than inventing text.
183
+ - **History walking assumes damage, once introduced, persists.** That assumption is verified,
184
+ and the search falls back to a linear scan when a file was broken, fixed, and broken again.
185
+ - Files over 5 MB are skipped by default.
186
+
187
+ ## How it works
188
+
189
+ ```
190
+ detect/facts.py BOM, line endings, strict UTF-8 decode - facts, no inference
191
+ detect/codecs.py D1: which legacy codec, and is the transcode reversible
192
+ detect/mojibake.py D2: repair plan, layer depth, and the inverse-plan proof
193
+ gitfx/repo.py git, always via bytes - never text=True, never the console codepage
194
+ gitfx/forensics.py binary-searches history for the commit that introduced the damage
195
+ gitfx/recover.py splices destroyed runs from the clean version, keeping later edits
196
+ fixing/journal.py gzipped originals so undo is byte-identical
197
+ hooks/ pre-commit entry point and the agent PreToolUse guard
198
+ ```
199
+
200
+ One detail worth calling out: `gitfx/repo.py` never uses `subprocess(text=True)`. That decodes
201
+ with the console codepage — cp1254 on a Turkish Windows box — which is precisely the bug class
202
+ this tool hunts. A forensics tool that mangles its own evidence is worthless, so every git call
203
+ returns bytes and decoding is always an explicit, local decision.
204
+
205
+ ## Why this exists
206
+
207
+ A memory system I run injected context into every session for months. It was writing `y���n�`
208
+ where it meant `yönetici`, and nothing ever failed — the sessions just quietly got a little
209
+ worse. The cause was a PowerShell 5.1 hook writing stdout in the console codepage.
210
+
211
+ Finding it took an afternoon. Knowing *when* it started took git archaeology by hand, and by
212
+ then the answer to "can any of this be recovered?" was pure guesswork. mojiblame is that
213
+ afternoon, made repeatable.
214
+
215
+ Its first real run, against that same 437-file vault, found four damaged files and named the
216
+ commit responsible for each.
217
+
218
+ ## License
219
+
220
+ MIT
Binary file
@@ -0,0 +1,77 @@
1
+ <svg xmlns="http://www.w3.org/2000/svg" width="1400" height="430" viewBox="0 0 1400 430" role="img" aria-labelledby="title desc">
2
+ <!-- mojiblame:ignore-file — the final node intentionally shows a mojibake sample. -->
3
+ <title id="title">How silent encoding corruption enters a repository</title>
4
+ <desc id="desc">A UTF-8 file is read by an agent or tool through a shell, decoded with the wrong codepage, and written back as mojibake. Every step reports success.</desc>
5
+ <defs>
6
+ <linearGradient id="panel" x1="0" y1="0" x2="1" y2="1">
7
+ <stop offset="0" stop-color="#10171c"/>
8
+ <stop offset="1" stop-color="#080c0f"/>
9
+ </linearGradient>
10
+ <filter id="shadow" x="-20%" y="-20%" width="140%" height="140%">
11
+ <feDropShadow dx="0" dy="8" stdDeviation="10" flood-color="#000" flood-opacity=".28"/>
12
+ </filter>
13
+ <marker id="arrow" viewBox="0 0 10 10" refX="9" refY="5" markerWidth="7" markerHeight="7" orient="auto-start-reverse">
14
+ <path d="M 0 0 L 10 5 L 0 10 z" fill="#93a1aa"/>
15
+ </marker>
16
+ <style>
17
+ .label { font: 700 14px ui-monospace, SFMono-Regular, Consolas, monospace; letter-spacing: .12em; fill: #ffab00; }
18
+ .title { font: 700 21px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #edf1f3; }
19
+ .body { font: 400 16px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #aeb9bf; }
20
+ .code { font: 600 18px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #edf1f3; }
21
+ .arrow-label { font: 600 13px ui-monospace, SFMono-Regular, Consolas, monospace; fill: #7f8c94; letter-spacing: .08em; }
22
+ </style>
23
+ </defs>
24
+
25
+ <rect x="1" y="1" width="1398" height="428" rx="18" fill="url(#panel)" stroke="#34424a" stroke-width="2"/>
26
+ <text x="54" y="51" class="label">THE SILENT CORRUPTION PATH</text>
27
+ <text x="1346" y="51" text-anchor="end" class="body">deterministic · no inference</text>
28
+
29
+ <g filter="url(#shadow)">
30
+ <g transform="translate(54 92)">
31
+ <rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
32
+ <text x="24" y="36" class="label">01 · FILE</text>
33
+ <text x="24" y="76" class="title">UTF-8 on disk</text>
34
+ <rect x="24" y="100" width="212" height="48" rx="7" fill="#080c0f" stroke="#2c3940"/>
35
+ <text x="42" y="131" class="code">yönetici</text>
36
+ </g>
37
+
38
+ <g transform="translate(397 92)">
39
+ <rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
40
+ <text x="24" y="36" class="label">02 · READ</text>
41
+ <text x="24" y="76" class="title">agent / tool</text>
42
+ <text x="24" y="111" class="body">shell</text>
43
+ <text x="24" y="140" class="body">subprocess stdout</text>
44
+ </g>
45
+
46
+ <g transform="translate(740 92)">
47
+ <rect width="260" height="176" rx="13" fill="#1c1710" stroke="#ffab00" stroke-width="2"/>
48
+ <text x="24" y="36" class="label">03 · DECODE</text>
49
+ <text x="24" y="76" class="title">wrong codepage</text>
50
+ <text x="24" y="113" class="code">cp1254</text>
51
+ <text x="24" y="143" class="body">bytes ≠ characters</text>
52
+ </g>
53
+
54
+ <g transform="translate(1083 92)">
55
+ <rect width="260" height="176" rx="13" fill="#121b20" stroke="#44535b"/>
56
+ <text x="24" y="36" class="label">04 · WRITE BACK</text>
57
+ <text x="24" y="76" class="title">valid UTF-8</text>
58
+ <rect x="24" y="100" width="212" height="48" rx="7" fill="#080c0f" stroke="#ffab00"/>
59
+ <text x="42" y="131" class="code" fill="#ffbf3f">yönetici</text>
60
+ </g>
61
+ </g>
62
+
63
+ <g fill="none" stroke="#93a1aa" stroke-width="2" marker-end="url(#arrow)">
64
+ <path d="M 320 180 H 381"/>
65
+ <path d="M 663 180 H 724"/>
66
+ <path d="M 1006 180 H 1067"/>
67
+ </g>
68
+ <text x="350" y="164" text-anchor="middle" class="arrow-label">READ</text>
69
+ <text x="693" y="164" text-anchor="middle" class="arrow-label">TEXT</text>
70
+ <text x="1036" y="164" text-anchor="middle" class="arrow-label">WRITE</text>
71
+
72
+ <rect x="54" y="309" width="1289" height="76" rx="12" fill="#18140c" stroke="#ffab00" stroke-width="2"/>
73
+ <circle cx="91" cy="347" r="17" fill="#ffab00"/>
74
+ <path d="M 91 335 V 351 M 91 361 V 362" stroke="#111" stroke-width="4" stroke-linecap="round"/>
75
+ <text x="126" y="341" class="title">NO STEP THROWS AN ERROR.</text>
76
+ <text x="126" y="369" class="body">Read succeeds. Decode succeeds. Write succeeds. The silence is the bug.</text>
77
+ </svg>