jp-receipt-normalize 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- jp_receipt_normalize-0.1.0/.coverage +0 -0
- jp_receipt_normalize-0.1.0/.github/workflows/publish.yaml +40 -0
- jp_receipt_normalize-0.1.0/CHANGELOG.md +65 -0
- jp_receipt_normalize-0.1.0/LICENSE +21 -0
- jp_receipt_normalize-0.1.0/PKG-INFO +225 -0
- jp_receipt_normalize-0.1.0/README.md +196 -0
- jp_receipt_normalize-0.1.0/README_en.md +104 -0
- jp_receipt_normalize-0.1.0/docs/design_principles.md +57 -0
- jp_receipt_normalize-0.1.0/jpdoc_normalize_request.md +210 -0
- jp_receipt_normalize-0.1.0/pyproject.toml +62 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/__init__.py +35 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/char_maps.py +92 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/__init__.py +17 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/amount.py +184 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/date.py +236 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/jan.py +107 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/phone.py +129 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/registration.py +118 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/fields/time.py +148 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/models.py +113 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/normalizer.py +238 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/py.typed +0 -0
- jp_receipt_normalize-0.1.0/src/jpdoc_normalize/utils.py +89 -0
- jp_receipt_normalize-0.1.0/tests/fixtures/synthetic_receipts.json +37 -0
- jp_receipt_normalize-0.1.0/tests/test_amount.py +49 -0
- jp_receipt_normalize-0.1.0/tests/test_coverage_edge_cases.py +67 -0
- jp_receipt_normalize-0.1.0/tests/test_date.py +58 -0
- jp_receipt_normalize-0.1.0/tests/test_fixtures.py +63 -0
- jp_receipt_normalize-0.1.0/tests/test_jan.py +39 -0
- jp_receipt_normalize-0.1.0/tests/test_models.py +64 -0
- jp_receipt_normalize-0.1.0/tests/test_normalizer.py +67 -0
- jp_receipt_normalize-0.1.0/tests/test_phone.py +29 -0
- jp_receipt_normalize-0.1.0/tests/test_registration.py +44 -0
- jp_receipt_normalize-0.1.0/tests/test_time.py +45 -0
|
Binary file
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
name: Publish to TestPyPI and PyPI
|
|
2
|
+
|
|
3
|
+
on:
|
|
4
|
+
push:
|
|
5
|
+
tags:
|
|
6
|
+
- 'v*' # v0.1.0 などのタグが付いたときだけ実行
|
|
7
|
+
|
|
8
|
+
jobs:
|
|
9
|
+
publish:
|
|
10
|
+
runs-on: ubuntu-latest
|
|
11
|
+
permissions:
|
|
12
|
+
id-token: write # OIDC トークン取得に必須
|
|
13
|
+
steps:
|
|
14
|
+
- name: Checkout repository
|
|
15
|
+
uses: actions/checkout@v4
|
|
16
|
+
|
|
17
|
+
- name: Set up Python
|
|
18
|
+
uses: actions/setup-python@v5
|
|
19
|
+
with:
|
|
20
|
+
python-version: '3.12'
|
|
21
|
+
|
|
22
|
+
- name: Install build tools
|
|
23
|
+
run: |
|
|
24
|
+
python -m pip install --upgrade pip
|
|
25
|
+
pip install build twine
|
|
26
|
+
|
|
27
|
+
- name: Build package
|
|
28
|
+
run: python -m build
|
|
29
|
+
|
|
30
|
+
- name: Publish to TestPyPI
|
|
31
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
32
|
+
with:
|
|
33
|
+
repository-url: https://test.pypi.org/legacy/
|
|
34
|
+
skip-existing: true
|
|
35
|
+
|
|
36
|
+
- name: Clean up TestPyPI attestations
|
|
37
|
+
run: rm -f dist/*.publish.attestation
|
|
38
|
+
|
|
39
|
+
- name: Publish to PyPI
|
|
40
|
+
uses: pypa/gh-action-pypi-publish@release/v1
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
All notable changes to `jp-receipt-normalize` will be documented in this file.
|
|
4
|
+
This project adheres to [Semantic Versioning](https://semver.org/).
|
|
5
|
+
|
|
6
|
+
## [0.1.0] - 2026-11-15
|
|
7
|
+
|
|
8
|
+
### Added
|
|
9
|
+
- **Core Models**:
|
|
10
|
+
- `SourceSpan`: Preserves original OCR text, coordinates, bounding box, and confidence.
|
|
11
|
+
- `Edit`: Immutable audit trail tracking transformation rule, source substring, replacement, and start/end offsets.
|
|
12
|
+
- `NormalizedText`: Container for normalized string, original SourceSpan, and tuple of edits.
|
|
13
|
+
- `Candidate[T]`: Structured extraction candidate with value, source span, edits, status (`exact`, `normalized`, `ambiguous`, `invalid`), and reasons.
|
|
14
|
+
- Full JSON serialization support via `.to_dict()` and `.from_dict()`.
|
|
15
|
+
- **Auditable Normalization (`normalize_text`)**:
|
|
16
|
+
- Fullwidth/halfwidth alphanumeric and symbol unification based on NFKC principles without silent character loss.
|
|
17
|
+
- Explicit handling and removal of zero-width and invisible control characters.
|
|
18
|
+
- Whitespace normalization (fullwidth space `\u3000`, NBSP `\u00A0`, narrow spaces).
|
|
19
|
+
- Profiles: `generic`, `receipt`, and `invoice`.
|
|
20
|
+
- Han/CJK simplified and variant character mapping for OCR misreads (`计`->`計`, `圆`/`圓`->`円`, `总`->`総`, `额`->`額`, `领`->`領`, `收`->`収`, `证`->`証`, etc.).
|
|
21
|
+
- Contextual hyphen/prolonged-sound distinction (protects Katakana words like `コーヒー` from illegal hyphenation).
|
|
22
|
+
- **Field Candidate Extractors (`jpdoc_normalize.fields`)**:
|
|
23
|
+
- `find_amounts`: Money amounts with Yen signs, negative indicators (▲, △, -, accounting parens), tax context, and period/comma ambiguity detection.
|
|
24
|
+
- `find_dates`: Western calendar (4-digit and 2-digit) and Japanese Wareki (令和, 平成, 昭和, 元年), with strict calendar existence validation (leap years, nonexistent dates).
|
|
25
|
+
- `find_times`: 24-hour time representation with OCR colon variations (`;`, `.`) and Japanese character format (`時分秒`).
|
|
26
|
+
- `find_phone_numbers`: Japanese phone number extraction (landlines, mobile 070/080/090, IP 050, toll-free 0120/0800, navidial 0570, data 020) with hyphen stripping.
|
|
27
|
+
- `find_invoice_registration_numbers`: Qualified invoice issuer registration number (`T` + 13 digits) with National Tax Agency (NTA) corporate number check digit verification.
|
|
28
|
+
- `find_jan_codes`: JAN-8 and JAN-13 barcode candidate extraction with Modulus 10 Weight 3 check digit verification.
|
|
29
|
+
- **Fixtures & Tests**:
|
|
30
|
+
- Synthetic receipt and invoice fixture dataset (`synthetic_receipts.json`).
|
|
31
|
+
- Comprehensive unit test suite with 94% branch/statement coverage.
|
|
32
|
+
- Strict type hints with `py.typed` and zero mypy errors under strict mode.
|
|
33
|
+
|
|
34
|
+
---
|
|
35
|
+
|
|
36
|
+
## 移行ガイド (Migration Guide)
|
|
37
|
+
|
|
38
|
+
### 既存の個別正規化スクリプトからの移行
|
|
39
|
+
|
|
40
|
+
既存の独自スクリプトで正規化を行っていた場合、以下の手順で安全に段階的移行が可能です:
|
|
41
|
+
|
|
42
|
+
1. **文字列正規化の置き換え**:
|
|
43
|
+
```python
|
|
44
|
+
# 従来: 単純なNFKCやreplace
|
|
45
|
+
# text = unicodedata.normalize("NFKC", raw_text)
|
|
46
|
+
|
|
47
|
+
# 移行後: 変更履歴が追跡可能な normalize_text
|
|
48
|
+
from jpdoc_normalize import normalize_text
|
|
49
|
+
normalized = normalize_text(raw_text, profile="receipt")
|
|
50
|
+
text = normalized.text
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
2. **フィールド抽出の導入**:
|
|
54
|
+
```python
|
|
55
|
+
# 従来: 個別の正規表現による手動パース
|
|
56
|
+
# 移行後: 監査可能なCandidate抽出
|
|
57
|
+
from jpdoc_normalize.fields import find_amounts, find_dates, find_invoice_registration_numbers
|
|
58
|
+
|
|
59
|
+
amounts = find_amounts(normalized)
|
|
60
|
+
dates = find_dates(normalized)
|
|
61
|
+
registrations = find_invoice_registration_numbers(normalized)
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
3. **安全フラグ(Feature Flag)による並行稼働**:
|
|
65
|
+
- `Candidate.status` が `"ambiguous"` または `"invalid"` の場合は人間オペレーターのレビューに回すか、既存のフォールバックロジックを呼び出す設計にすることで、誤補正リスクをゼロに抑えられます。
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 jp-receipt-normalize contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: jp-receipt-normalize
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Auditable normalization and candidate extraction for Japanese document OCR
|
|
5
|
+
Project-URL: Homepage, https://github.com/IT-Engineer-K/jp-receipt-normalize
|
|
6
|
+
Project-URL: Repository, https://github.com/IT-Engineer-K/jp-receipt-normalize
|
|
7
|
+
Project-URL: Changelog, https://github.com/IT-Engineer-K/jp-receipt-normalize/blob/main/CHANGELOG.md
|
|
8
|
+
Author: jp-receipt-normalize contributors
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Text Processing :: General
|
|
21
|
+
Classifier: Typing :: Typed
|
|
22
|
+
Requires-Python: >=3.10
|
|
23
|
+
Provides-Extra: dev
|
|
24
|
+
Requires-Dist: mypy>=1.0.0; extra == 'dev'
|
|
25
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == 'dev'
|
|
26
|
+
Requires-Dist: pytest>=7.0.0; extra == 'dev'
|
|
27
|
+
Requires-Dist: ruff>=0.1.0; extra == 'dev'
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
|
|
30
|
+
# jp-receipt-normalize
|
|
31
|
+
|
|
32
|
+
[](https://www.python.org/downloads/)
|
|
33
|
+
[](https://opensource.org/licenses/MIT)
|
|
34
|
+
[](https://peps.python.org/pep-0561/)
|
|
35
|
+
|
|
36
|
+
**日本語帳票OCRの監査可能な正規化・候補抽出ライブラリ**
|
|
37
|
+
|
|
38
|
+
[English README is available here (README_en.md)](README_en.md) | [設計思想: なぜ曖昧なら直さないか (docs/design_principles.md)](docs/design_principles.md)
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## 概要
|
|
43
|
+
|
|
44
|
+
日本語の領収書・レシート・請求書をOCR処理すると、全角半角、Unicode字形、中国語字形(簡体字)、記号の誤読、空白、桁区切りなどが揺れます。
|
|
45
|
+
|
|
46
|
+
しかし、業務自動化において「それらしく見える値へ勝手に直す(サイレント確定)」ことは、誤請求や税務申告ミスを引き起こす極めて危険な行為です。
|
|
47
|
+
|
|
48
|
+
`jp-receipt-normalize` は、**OCRエンジンやLLMを一切内包せず**、OCR後段の処理に特化した監査可能(Auditable)な部品です。
|
|
49
|
+
元の文字列、変更箇所、適用規則、曖昧性の理由をすべて記録・保持し、安全な候補抽出を提供します。
|
|
50
|
+
|
|
51
|
+
### 特徴
|
|
52
|
+
|
|
53
|
+
1. **完全な監査可能性(Auditability)**:
|
|
54
|
+
すべての文字列変更は `Edit(rule, source, replacement, start, end)` として記録され、元の位置と差分を100%追跡できます。
|
|
55
|
+
2. **サイレント確定の排除**:
|
|
56
|
+
「O」と「0」、「l」と「1」、ピリオド桁区切り(`¥1.840`)など、文脈なしに断定できない曖昧な補正は自動確定せず、`status="ambiguous"` や判定理由(`reasons`)を返します。
|
|
57
|
+
3. **日本語帳票に特化した候補抽出**:
|
|
58
|
+
- 金額(負数・値引き・税額・通貨記号・桁区切り)
|
|
59
|
+
- 日付(西暦・和暦・令和元年・実在日うるう年検証)
|
|
60
|
+
- 時刻(24時間表記・時分秒・コロン揺れ)
|
|
61
|
+
- 日本の電話番号(市外局番体系・固定・携帯・フリーダイヤル・IP電話・ハイフン除去)
|
|
62
|
+
- 適格請求書発行事業者登録番号(`T` + 13桁・国税庁法人番号チェックデジット検証)
|
|
63
|
+
- JAN/EANコード(8桁・13桁・モジュラス10ウェイト3チェックデジット検証)
|
|
64
|
+
4. **外部通信なし・ゼロ依存**:
|
|
65
|
+
Python標準ライブラリ中心で動作。外部API呼び出しやネットワーク通信は一切行わず、決定論的(Deterministic)に動作します。
|
|
66
|
+
5. **JSON直列化可能 & 厳格な型付け**:
|
|
67
|
+
`to_dict()` による完全なJSON対応と、`py.typed` による静的型チェック(mypy strict)に対応。
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## インストール
|
|
72
|
+
|
|
73
|
+
```bash
|
|
74
|
+
pip install jp-receipt-normalize
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## クイックスタート
|
|
80
|
+
|
|
81
|
+
```python
|
|
82
|
+
from jpdoc_normalize import normalize_text
|
|
83
|
+
from jpdoc_normalize.fields import (
|
|
84
|
+
find_amounts,
|
|
85
|
+
find_dates,
|
|
86
|
+
find_invoice_registration_numbers,
|
|
87
|
+
find_phone_numbers,
|
|
88
|
+
)
|
|
89
|
+
|
|
90
|
+
# 1. 文字列の監査可能正規化
|
|
91
|
+
ocr_text = "合 計 ¥1,840"
|
|
92
|
+
normalized = normalize_text(ocr_text, profile="receipt")
|
|
93
|
+
|
|
94
|
+
print(normalized.text)
|
|
95
|
+
# => "合 計 ¥1,840"
|
|
96
|
+
|
|
97
|
+
# 変更履歴(Edit)の確認
|
|
98
|
+
for edit in normalized.edits:
|
|
99
|
+
print(f"Rule: {edit.rule}, Source: {edit.source!r} -> {edit.replacement!r} [{edit.start}:{edit.end}]")
|
|
100
|
+
# => Rule: normalize_space, Source: ' ' -> ' ' [2:3]
|
|
101
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '8' -> '8' [5:6]
|
|
102
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '4' -> '4' [6:7]
|
|
103
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '0' -> '0' [7:8]
|
|
104
|
+
|
|
105
|
+
# 2. 金額候補の抽出
|
|
106
|
+
amounts = find_amounts(normalized)
|
|
107
|
+
print(amounts[0].value, amounts[0].status, amounts[0].reasons)
|
|
108
|
+
# => 1840 normalized ('has_currency_symbol',)
|
|
109
|
+
|
|
110
|
+
# 3. インボイス登録番号の抽出と法人番号チェックデジット検証
|
|
111
|
+
registrations = find_invoice_registration_numbers("登録番号 T6120001018791")
|
|
112
|
+
for candidate in registrations:
|
|
113
|
+
print(candidate.value, candidate.status, candidate.reasons)
|
|
114
|
+
# => T6120001018791 normalized ('has_registration_keyword', 'valid_corporate_number_checksum')
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
## 主要データ型
|
|
120
|
+
|
|
121
|
+
### `SourceSpan`
|
|
122
|
+
元のOCR入力におけるテキストスパン、座標情報、信頼度を保持します。
|
|
123
|
+
|
|
124
|
+
```python
|
|
125
|
+
@dataclass(frozen=True)
|
|
126
|
+
class SourceSpan:
|
|
127
|
+
text: str
|
|
128
|
+
start: int
|
|
129
|
+
end: int
|
|
130
|
+
page: int | None = None
|
|
131
|
+
bbox: tuple[float, float, float, float] | None = None
|
|
132
|
+
confidence: float | None = None
|
|
133
|
+
engine: str | None = None
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### `Edit`
|
|
137
|
+
テキスト正規化における1つの編集操作。監査証跡として保存されます。
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
@dataclass(frozen=True)
|
|
141
|
+
class Edit:
|
|
142
|
+
rule: str
|
|
143
|
+
source: str
|
|
144
|
+
replacement: str
|
|
145
|
+
start: int
|
|
146
|
+
end: int
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
### `Candidate[T]`
|
|
150
|
+
抽出されたフィールドの候補値。判定ステータスと根拠理由を持ちます。
|
|
151
|
+
|
|
152
|
+
```python
|
|
153
|
+
@dataclass(frozen=True)
|
|
154
|
+
class Candidate(Generic[T]):
|
|
155
|
+
value: T
|
|
156
|
+
source: SourceSpan
|
|
157
|
+
edits: tuple[Edit, ...]
|
|
158
|
+
status: Literal["exact", "normalized", "ambiguous", "invalid"]
|
|
159
|
+
reasons: tuple[str, ...]
|
|
160
|
+
```
|
|
161
|
+
|
|
162
|
+
#### 判定ステータス (`Candidate.status`)
|
|
163
|
+
- `exact`: 原文のまま有効な値(カンマなし数字など)
|
|
164
|
+
- `normalized`: 非曖昧な正規化(全角半角変換、カンマ除去等)を経て有効な値
|
|
165
|
+
- `ambiguous`: 複数解釈が可能(2桁西暦、ピリオドによる桁区切りの疑い等)
|
|
166
|
+
- `invalid`: チェックデジット不一致、存在しない実在日など形式的に不正
|
|
167
|
+
|
|
168
|
+
---
|
|
169
|
+
|
|
170
|
+
## フィールド抽出機能一覧
|
|
171
|
+
|
|
172
|
+
| 関数名 | 抽出対象 | 主な対応内容・検証 |
|
|
173
|
+
|---|---|---|
|
|
174
|
+
| `find_amounts` | 金額 | 円表記(`¥`, `¥`, `円`)、3桁区切り、値引き・負数(`▲`, `△`, `-`, 会計括弧 `(1,000)`)、ピリオド混入の曖昧性検出 |
|
|
175
|
+
| `find_dates` | 日付 | 西暦(4桁・2桁)、和暦(令和、平成、昭和、元年、略記 `R8.11.15`)、実在日検証(うるう年、存在しない日の除外) |
|
|
176
|
+
| `find_times` | 時刻 | 24時間表記(`17:30`, `17:30:15`)、日本語表記(`17時30分`)、コロンOCR揺れ(`;`, `.`) |
|
|
177
|
+
| `find_phone_numbers` | 電話番号 | 日本の固定電話、携帯電話(070/080/090)、IP電話(050)、フリーダイヤル(0120/0800)、ハイフン・空白除去 |
|
|
178
|
+
| `find_invoice_registration_numbers` | 適格請求書登録番号 | `T` + 13桁、国税庁法人番号チェックデジット(モジュラス9 交互ウェイト)検証 |
|
|
179
|
+
| `find_jan_codes` | JAN/EANコード | JAN-8 / JAN-13、GS1規格モジュラス10 ウェイト3チェックデジット検証 |
|
|
180
|
+
|
|
181
|
+
---
|
|
182
|
+
|
|
183
|
+
## プロファイル
|
|
184
|
+
|
|
185
|
+
`normalize_text` には以下のプロファイルを指定できます:
|
|
186
|
+
|
|
187
|
+
- `generic`: 基本的な全角半角統一、不可視文字・空白の正規化。
|
|
188
|
+
- `receipt`: レシート頻出の通貨記号、簡体字・異体字辞書(`圆`・`圓`・`元` -> `円`、`计` -> `計`、`总` -> `総` など)を適用。
|
|
189
|
+
- `invoice`: 請求書特有のインボイス用語・異体字辞書を適用。
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## 開発とテスト
|
|
194
|
+
|
|
195
|
+
本ライブラリは、高いテストカバレッジと型安全性を重視しています。
|
|
196
|
+
|
|
197
|
+
```bash
|
|
198
|
+
# テストの実行
|
|
199
|
+
pytest
|
|
200
|
+
|
|
201
|
+
# カバレッジレポート
|
|
202
|
+
pytest --cov=jpdoc_normalize --cov-report=term-missing
|
|
203
|
+
|
|
204
|
+
# 型検査
|
|
205
|
+
mypy src
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## 設計思想と非目標
|
|
211
|
+
|
|
212
|
+
本ライブラリは以下の「非目標」を明確に定めています:
|
|
213
|
+
|
|
214
|
+
- **非目標**: OCR推論そのもの、画像傾き補正、レイアウト解析
|
|
215
|
+
- **非目標**: LLMによる「おそらくこういう意味だろう」という文脈補完
|
|
216
|
+
- **非目標**: 帳票全体を1回でJSON化するエンドツーエンド製品
|
|
217
|
+
- **非目標**: 「どの日付・どの金額を採用するか」という業務意思決定
|
|
218
|
+
|
|
219
|
+
詳細は [設計思想ドキュメント (docs/design_principles.md)](docs/design_principles.md) をご覧ください。
|
|
220
|
+
|
|
221
|
+
---
|
|
222
|
+
|
|
223
|
+
## ライセンス
|
|
224
|
+
|
|
225
|
+
[MIT License](LICENSE)
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
# jp-receipt-normalize
|
|
2
|
+
|
|
3
|
+
[](https://www.python.org/downloads/)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://peps.python.org/pep-0561/)
|
|
6
|
+
|
|
7
|
+
**日本語帳票OCRの監査可能な正規化・候補抽出ライブラリ**
|
|
8
|
+
|
|
9
|
+
[English README is available here (README_en.md)](README_en.md) | [設計思想: なぜ曖昧なら直さないか (docs/design_principles.md)](docs/design_principles.md)
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## 概要
|
|
14
|
+
|
|
15
|
+
日本語の領収書・レシート・請求書をOCR処理すると、全角半角、Unicode字形、中国語字形(簡体字)、記号の誤読、空白、桁区切りなどが揺れます。
|
|
16
|
+
|
|
17
|
+
しかし、業務自動化において「それらしく見える値へ勝手に直す(サイレント確定)」ことは、誤請求や税務申告ミスを引き起こす極めて危険な行為です。
|
|
18
|
+
|
|
19
|
+
`jp-receipt-normalize` は、**OCRエンジンやLLMを一切内包せず**、OCR後段の処理に特化した監査可能(Auditable)な部品です。
|
|
20
|
+
元の文字列、変更箇所、適用規則、曖昧性の理由をすべて記録・保持し、安全な候補抽出を提供します。
|
|
21
|
+
|
|
22
|
+
### 特徴
|
|
23
|
+
|
|
24
|
+
1. **完全な監査可能性(Auditability)**:
|
|
25
|
+
すべての文字列変更は `Edit(rule, source, replacement, start, end)` として記録され、元の位置と差分を100%追跡できます。
|
|
26
|
+
2. **サイレント確定の排除**:
|
|
27
|
+
「O」と「0」、「l」と「1」、ピリオド桁区切り(`¥1.840`)など、文脈なしに断定できない曖昧な補正は自動確定せず、`status="ambiguous"` や判定理由(`reasons`)を返します。
|
|
28
|
+
3. **日本語帳票に特化した候補抽出**:
|
|
29
|
+
- 金額(負数・値引き・税額・通貨記号・桁区切り)
|
|
30
|
+
- 日付(西暦・和暦・令和元年・実在日うるう年検証)
|
|
31
|
+
- 時刻(24時間表記・時分秒・コロン揺れ)
|
|
32
|
+
- 日本の電話番号(市外局番体系・固定・携帯・フリーダイヤル・IP電話・ハイフン除去)
|
|
33
|
+
- 適格請求書発行事業者登録番号(`T` + 13桁・国税庁法人番号チェックデジット検証)
|
|
34
|
+
- JAN/EANコード(8桁・13桁・モジュラス10ウェイト3チェックデジット検証)
|
|
35
|
+
4. **外部通信なし・ゼロ依存**:
|
|
36
|
+
Python標準ライブラリ中心で動作。外部API呼び出しやネットワーク通信は一切行わず、決定論的(Deterministic)に動作します。
|
|
37
|
+
5. **JSON直列化可能 & 厳格な型付け**:
|
|
38
|
+
`to_dict()` による完全なJSON対応と、`py.typed` による静的型チェック(mypy strict)に対応。
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## インストール
|
|
43
|
+
|
|
44
|
+
```bash
|
|
45
|
+
pip install jp-receipt-normalize
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
---
|
|
49
|
+
|
|
50
|
+
## クイックスタート
|
|
51
|
+
|
|
52
|
+
```python
|
|
53
|
+
from jpdoc_normalize import normalize_text
|
|
54
|
+
from jpdoc_normalize.fields import (
|
|
55
|
+
find_amounts,
|
|
56
|
+
find_dates,
|
|
57
|
+
find_invoice_registration_numbers,
|
|
58
|
+
find_phone_numbers,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
# 1. 文字列の監査可能正規化
|
|
62
|
+
ocr_text = "合 計 ¥1,840"
|
|
63
|
+
normalized = normalize_text(ocr_text, profile="receipt")
|
|
64
|
+
|
|
65
|
+
print(normalized.text)
|
|
66
|
+
# => "合 計 ¥1,840"
|
|
67
|
+
|
|
68
|
+
# 変更履歴(Edit)の確認
|
|
69
|
+
for edit in normalized.edits:
|
|
70
|
+
print(f"Rule: {edit.rule}, Source: {edit.source!r} -> {edit.replacement!r} [{edit.start}:{edit.end}]")
|
|
71
|
+
# => Rule: normalize_space, Source: ' ' -> ' ' [2:3]
|
|
72
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '8' -> '8' [5:6]
|
|
73
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '4' -> '4' [6:7]
|
|
74
|
+
# => Rule: fullwidth_digit_to_halfwidth, Source: '0' -> '0' [7:8]
|
|
75
|
+
|
|
76
|
+
# 2. 金額候補の抽出
|
|
77
|
+
amounts = find_amounts(normalized)
|
|
78
|
+
print(amounts[0].value, amounts[0].status, amounts[0].reasons)
|
|
79
|
+
# => 1840 normalized ('has_currency_symbol',)
|
|
80
|
+
|
|
81
|
+
# 3. インボイス登録番号の抽出と法人番号チェックデジット検証
|
|
82
|
+
registrations = find_invoice_registration_numbers("登録番号 T6120001018791")
|
|
83
|
+
for candidate in registrations:
|
|
84
|
+
print(candidate.value, candidate.status, candidate.reasons)
|
|
85
|
+
# => T6120001018791 normalized ('has_registration_keyword', 'valid_corporate_number_checksum')
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
---
|
|
89
|
+
|
|
90
|
+
## 主要データ型
|
|
91
|
+
|
|
92
|
+
### `SourceSpan`
|
|
93
|
+
元のOCR入力におけるテキストスパン、座標情報、信頼度を保持します。
|
|
94
|
+
|
|
95
|
+
```python
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class SourceSpan:
|
|
98
|
+
text: str
|
|
99
|
+
start: int
|
|
100
|
+
end: int
|
|
101
|
+
page: int | None = None
|
|
102
|
+
bbox: tuple[float, float, float, float] | None = None
|
|
103
|
+
confidence: float | None = None
|
|
104
|
+
engine: str | None = None
|
|
105
|
+
```
|
|
106
|
+
|
|
107
|
+
### `Edit`
|
|
108
|
+
テキスト正規化における1つの編集操作。監査証跡として保存されます。
|
|
109
|
+
|
|
110
|
+
```python
|
|
111
|
+
@dataclass(frozen=True)
|
|
112
|
+
class Edit:
|
|
113
|
+
rule: str
|
|
114
|
+
source: str
|
|
115
|
+
replacement: str
|
|
116
|
+
start: int
|
|
117
|
+
end: int
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
### `Candidate[T]`
|
|
121
|
+
抽出されたフィールドの候補値。判定ステータスと根拠理由を持ちます。
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
@dataclass(frozen=True)
|
|
125
|
+
class Candidate(Generic[T]):
|
|
126
|
+
value: T
|
|
127
|
+
source: SourceSpan
|
|
128
|
+
edits: tuple[Edit, ...]
|
|
129
|
+
status: Literal["exact", "normalized", "ambiguous", "invalid"]
|
|
130
|
+
reasons: tuple[str, ...]
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
#### 判定ステータス (`Candidate.status`)
|
|
134
|
+
- `exact`: 原文のまま有効な値(カンマなし数字など)
|
|
135
|
+
- `normalized`: 非曖昧な正規化(全角半角変換、カンマ除去等)を経て有効な値
|
|
136
|
+
- `ambiguous`: 複数解釈が可能(2桁西暦、ピリオドによる桁区切りの疑い等)
|
|
137
|
+
- `invalid`: チェックデジット不一致、存在しない実在日など形式的に不正
|
|
138
|
+
|
|
139
|
+
---
|
|
140
|
+
|
|
141
|
+
## フィールド抽出機能一覧
|
|
142
|
+
|
|
143
|
+
| 関数名 | 抽出対象 | 主な対応内容・検証 |
|
|
144
|
+
|---|---|---|
|
|
145
|
+
| `find_amounts` | 金額 | 円表記(`¥`, `¥`, `円`)、3桁区切り、値引き・負数(`▲`, `△`, `-`, 会計括弧 `(1,000)`)、ピリオド混入の曖昧性検出 |
|
|
146
|
+
| `find_dates` | 日付 | 西暦(4桁・2桁)、和暦(令和、平成、昭和、元年、略記 `R8.11.15`)、実在日検証(うるう年、存在しない日の除外) |
|
|
147
|
+
| `find_times` | 時刻 | 24時間表記(`17:30`, `17:30:15`)、日本語表記(`17時30分`)、コロンOCR揺れ(`;`, `.`) |
|
|
148
|
+
| `find_phone_numbers` | 電話番号 | 日本の固定電話、携帯電話(070/080/090)、IP電話(050)、フリーダイヤル(0120/0800)、ハイフン・空白除去 |
|
|
149
|
+
| `find_invoice_registration_numbers` | 適格請求書登録番号 | `T` + 13桁、国税庁法人番号チェックデジット(モジュラス9 交互ウェイト)検証 |
|
|
150
|
+
| `find_jan_codes` | JAN/EANコード | JAN-8 / JAN-13、GS1規格モジュラス10 ウェイト3チェックデジット検証 |
|
|
151
|
+
|
|
152
|
+
---
|
|
153
|
+
|
|
154
|
+
## プロファイル
|
|
155
|
+
|
|
156
|
+
`normalize_text` には以下のプロファイルを指定できます:
|
|
157
|
+
|
|
158
|
+
- `generic`: 基本的な全角半角統一、不可視文字・空白の正規化。
|
|
159
|
+
- `receipt`: レシート頻出の通貨記号、簡体字・異体字辞書(`圆`・`圓`・`元` -> `円`、`计` -> `計`、`总` -> `総` など)を適用。
|
|
160
|
+
- `invoice`: 請求書特有のインボイス用語・異体字辞書を適用。
|
|
161
|
+
|
|
162
|
+
---
|
|
163
|
+
|
|
164
|
+
## 開発とテスト
|
|
165
|
+
|
|
166
|
+
本ライブラリは、高いテストカバレッジと型安全性を重視しています。
|
|
167
|
+
|
|
168
|
+
```bash
|
|
169
|
+
# テストの実行
|
|
170
|
+
pytest
|
|
171
|
+
|
|
172
|
+
# カバレッジレポート
|
|
173
|
+
pytest --cov=jpdoc_normalize --cov-report=term-missing
|
|
174
|
+
|
|
175
|
+
# 型検査
|
|
176
|
+
mypy src
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
---
|
|
180
|
+
|
|
181
|
+
## 設計思想と非目標
|
|
182
|
+
|
|
183
|
+
本ライブラリは以下の「非目標」を明確に定めています:
|
|
184
|
+
|
|
185
|
+
- **非目標**: OCR推論そのもの、画像傾き補正、レイアウト解析
|
|
186
|
+
- **非目標**: LLMによる「おそらくこういう意味だろう」という文脈補完
|
|
187
|
+
- **非目標**: 帳票全体を1回でJSON化するエンドツーエンド製品
|
|
188
|
+
- **非目標**: 「どの日付・どの金額を採用するか」という業務意思決定
|
|
189
|
+
|
|
190
|
+
詳細は [設計思想ドキュメント (docs/design_principles.md)](docs/design_principles.md) をご覧ください。
|
|
191
|
+
|
|
192
|
+
---
|
|
193
|
+
|
|
194
|
+
## ライセンス
|
|
195
|
+
|
|
196
|
+
[MIT License](LICENSE)
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
# jp-receipt-normalize
|
|
2
|
+
|
|
3
|
+
[](https://www.python.org/downloads/)
|
|
4
|
+
[](https://opensource.org/licenses/MIT)
|
|
5
|
+
[](https://peps.python.org/pep-0561/)
|
|
6
|
+
|
|
7
|
+
**Auditable Normalization and Candidate Extraction for Japanese Document OCR**
|
|
8
|
+
|
|
9
|
+
[Japanese Documentation (README.md)](README.md) | [Design Principles (docs/design_principles.md)](docs/design_principles.md)
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Overview
|
|
14
|
+
|
|
15
|
+
When processing Japanese receipts, invoices, and financial documents with OCR, text often suffers from wide character variations, CJK Han glyph variants (Simplified Chinese characters leaking into Japanese OCR outputs), punctuation inconsistencies, and ambiguous numbers.
|
|
16
|
+
|
|
17
|
+
In automated business workflows (e.g. accounting, ERP, tax filing), **silent auto-corrections are dangerous**. A silent guess that turns an ambiguous character into the wrong digit can lead to compliance violations or incorrect tax reports.
|
|
18
|
+
|
|
19
|
+
`jp-receipt-normalize` does **not** contain any OCR engine or LLM. It is a lightweight, zero-dependency, and strictly auditable post-OCR processing component. It preserves raw strings, bounding boxes, character offsets, every single edit rule applied, and reasons for ambiguity.
|
|
20
|
+
|
|
21
|
+
### Key Features
|
|
22
|
+
|
|
23
|
+
1. **Complete Auditability**:
|
|
24
|
+
Every transformation is recorded as an `Edit(rule, source, replacement, start, end)`. You can trace 100% of character offsets back to the raw OCR text.
|
|
25
|
+
2. **No Silent Corrections**:
|
|
26
|
+
Ambiguous character replacements (such as letter `O` vs digit `0`, or `1.840` as period-thousands-separator vs decimal) are never silently committed; they are returned with `status="ambiguous"` and actionable reason tags.
|
|
27
|
+
3. **Japanese Document Field Extractors**:
|
|
28
|
+
- **Amounts**: Japanese Yen (`¥`, `¥`, `円`), negative amounts (▲, △, `-`, accounting parentheses `(1,000)`), discounts, tax contexts.
|
|
29
|
+
- **Dates**: Western calendar (4-digit and 2-digit) and Japanese Wareki (令和, 平成, 昭和, 元年), strictly validated against real calendar leap years.
|
|
30
|
+
- **Times**: 24-hour representations, Japanese characters (`17時30分`), OCR colon misreads (`;`, `.`).
|
|
31
|
+
- **Phone Numbers**: Japanese landlines, mobile (`070`, `080`, `090`), IP phones (`050`), toll-free (`0120`, `0800`), navidial (`0570`).
|
|
32
|
+
- **Qualified Invoice Issuer Registration Numbers**: `T` + 13 digits with Japanese National Tax Agency (NTA) corporate number check digit verification.
|
|
33
|
+
- **JAN/EAN Codes**: 8-digit and 13-digit barcodes with GS1 Modulus 10 Weight 3 check digit verification.
|
|
34
|
+
4. **Zero External Communication & Zero Dependencies**:
|
|
35
|
+
Built strictly with the Python standard library. Deterministic and fast.
|
|
36
|
+
5. **Full JSON Serialization & Strict Type Safety**:
|
|
37
|
+
Includes `.to_dict()` and `py.typed` passing `mypy --strict`.
|
|
38
|
+
|
|
39
|
+
---
|
|
40
|
+
|
|
41
|
+
## Installation
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
pip install jp-receipt-normalize
|
|
45
|
+
```
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## Quickstart
|
|
50
|
+
|
|
51
|
+
```python
|
|
52
|
+
from jpdoc_normalize import normalize_text
|
|
53
|
+
from jpdoc_normalize.fields import (
|
|
54
|
+
find_amounts,
|
|
55
|
+
find_dates,
|
|
56
|
+
find_invoice_registration_numbers,
|
|
57
|
+
find_phone_numbers,
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
# 1. Auditable text normalization
|
|
61
|
+
ocr_text = "合 計 ¥1,840"
|
|
62
|
+
normalized = normalize_text(ocr_text, profile="receipt")
|
|
63
|
+
|
|
64
|
+
print(normalized.text)
|
|
65
|
+
# => "合 計 ¥1,840"
|
|
66
|
+
|
|
67
|
+
# Inspect transformation edits
|
|
68
|
+
for edit in normalized.edits:
|
|
69
|
+
print(f"Rule: {edit.rule}, {edit.source!r} -> {edit.replacement!r} [{edit.start}:{edit.end}]")
|
|
70
|
+
|
|
71
|
+
# 2. Extract money amounts
|
|
72
|
+
amounts = find_amounts(normalized)
|
|
73
|
+
print(amounts[0].value, amounts[0].status, amounts[0].reasons)
|
|
74
|
+
# => 1840 normalized ('has_currency_symbol',)
|
|
75
|
+
|
|
76
|
+
# 3. Extract Invoice Registration Number with check digit validation
|
|
77
|
+
registrations = find_invoice_registration_numbers("登録番号 T6120001018791")
|
|
78
|
+
for candidate in registrations:
|
|
79
|
+
print(candidate.value, candidate.status, candidate.reasons)
|
|
80
|
+
# => T6120001018791 normalized ('has_registration_keyword', 'valid_corporate_number_checksum')
|
|
81
|
+
```
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## Core Data Models
|
|
86
|
+
|
|
87
|
+
- **`SourceSpan`**: Holds raw OCR string, coordinates, bounding box (`bbox`), and confidence.
|
|
88
|
+
- **`Edit`**: Represents a single immutable text substitution with rule name and character range.
|
|
89
|
+
- **`Candidate[T]`**: Extracted candidate value with `source`, `edits`, `status` (`exact`, `normalized`, `ambiguous`, `invalid`), and `reasons`.
|
|
90
|
+
|
|
91
|
+
---
|
|
92
|
+
|
|
93
|
+
## Non-Goals
|
|
94
|
+
|
|
95
|
+
`jp-receipt-normalize` deliberately excludes:
|
|
96
|
+
- OCR inference, image rotation, and layout analysis.
|
|
97
|
+
- LLM completion or speculative hallucination.
|
|
98
|
+
- End-to-end receipt-to-JSON business assumptions (e.g. deciding which of multiple dates is the due date).
|
|
99
|
+
|
|
100
|
+
---
|
|
101
|
+
|
|
102
|
+
## License
|
|
103
|
+
|
|
104
|
+
[MIT License](LICENSE)
|