panini-parser 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- panini_parser-0.1.0/MANIFEST.in +1 -0
- panini_parser-0.1.0/PKG-INFO +66 -0
- panini_parser-0.1.0/README.md +49 -0
- panini_parser-0.1.0/panini/__init__.py +12 -0
- panini_parser-0.1.0/panini/dates.py +50 -0
- panini_parser-0.1.0/panini/engine.py +497 -0
- panini_parser-0.1.0/panini/rules/archetypes.json +73 -0
- panini_parser-0.1.0/panini/rules/axis.json +62 -0
- panini_parser-0.1.0/panini/rules/bob.json +55 -0
- panini_parser-0.1.0/panini/rules/canara.json +39 -0
- panini_parser-0.1.0/panini/rules/categories.json +307 -0
- panini_parser-0.1.0/panini/rules/entities/ifsc-codes.json +53 -0
- panini_parser-0.1.0/panini/rules/entities/merchants.json +405 -0
- panini_parser-0.1.0/panini/rules/entities/vpa-handles.json +96 -0
- panini_parser-0.1.0/panini/rules/federal.json +41 -0
- panini_parser-0.1.0/panini/rules/generic.json +253 -0
- panini_parser-0.1.0/panini/rules/hdfc.json +53 -0
- panini_parser-0.1.0/panini/rules/icici.json +57 -0
- panini_parser-0.1.0/panini/rules/idfc.json +44 -0
- panini_parser-0.1.0/panini/rules/indusind.json +41 -0
- panini_parser-0.1.0/panini/rules/kotak.json +51 -0
- panini_parser-0.1.0/panini/rules/pnb.json +58 -0
- panini_parser-0.1.0/panini/rules/rbl.json +40 -0
- panini_parser-0.1.0/panini/rules/sbi.json +55 -0
- panini_parser-0.1.0/panini/rules/union.json +41 -0
- panini_parser-0.1.0/panini/rules/wallet.json +37 -0
- panini_parser-0.1.0/panini/rules/yes.json +40 -0
- panini_parser-0.1.0/panini/tokenizer.py +85 -0
- panini_parser-0.1.0/panini_parser.egg-info/PKG-INFO +66 -0
- panini_parser-0.1.0/panini_parser.egg-info/SOURCES.txt +34 -0
- panini_parser-0.1.0/panini_parser.egg-info/dependency_links.txt +1 -0
- panini_parser-0.1.0/panini_parser.egg-info/top_level.txt +1 -0
- panini_parser-0.1.0/pyproject.toml +32 -0
- panini_parser-0.1.0/setup.cfg +4 -0
- panini_parser-0.1.0/tests/test_bundled_rules.py +50 -0
- panini_parser-0.1.0/tests/test_conformance.py +56 -0
|
@@ -0,0 +1 @@
|
|
|
1
|
+
recursive-include panini/rules *.json
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: panini-parser
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Deterministic, privacy-preserving parser for Indian bank narrations — Python port of @panini/rules-engine
|
|
5
|
+
Author: People of Product
|
|
6
|
+
License: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/peopleofproduct/Panini
|
|
8
|
+
Project-URL: Repository, https://github.com/peopleofproduct/Panini
|
|
9
|
+
Keywords: india,fintech,bank-statement,upi,neft,imps,parser
|
|
10
|
+
Classifier: Development Status :: 3 - Alpha
|
|
11
|
+
Classifier: Intended Audience :: Developers
|
|
12
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Topic :: Office/Business :: Financial
|
|
15
|
+
Requires-Python: >=3.9
|
|
16
|
+
Description-Content-Type: text/markdown
|
|
17
|
+
|
|
18
|
+
# panini-py — Python port of Panini
|
|
19
|
+
|
|
20
|
+
Deterministic, privacy-preserving parser for Indian bank narrations.
|
|
21
|
+
This package is the Python port of `@paniniapi/rules-engine` and shares the
|
|
22
|
+
**same grammar JSONs and conformance corpus** — both runtimes are tested
|
|
23
|
+
against the identical fixture set, so parity is enforced, not assumed.
|
|
24
|
+
|
|
25
|
+
## Quick start
|
|
26
|
+
|
|
27
|
+
```python
|
|
28
|
+
from panini import parse_narration
|
|
29
|
+
|
|
30
|
+
txn = parse_narration("UPI/20261015/DR/826384910234/SWIGGY@okaxis/Cooking/")
|
|
31
|
+
# {
|
|
32
|
+
# "channel": "UPI", "type": "DEBIT", "ref_no": "826384910234",
|
|
33
|
+
# "merchant": "Swiggy", "vpa": "okaxis", "vpa_app": "Google Pay",
|
|
34
|
+
# "vpa_bank": "Axis Bank", "category": "Food & Dining", ...
|
|
35
|
+
# }
|
|
36
|
+
```
|
|
37
|
+
|
|
38
|
+
## Where the grammars live
|
|
39
|
+
|
|
40
|
+
The grammar JSONs ship **inside the wheel** under `panini/rules/` — a
|
|
41
|
+
pip install works standalone with no repo checkout. In a development
|
|
42
|
+
checkout the engine falls back to `packages/rules-engine/rules/`; the
|
|
43
|
+
bundled copy is kept byte-identical and drift is enforced by
|
|
44
|
+
`tests/test_bundled_rules.py`. Resolution order:
|
|
45
|
+
|
|
46
|
+
1. `PANINI_RULES_DIR` env var or `rules_dir=Path(...)` argument
|
|
47
|
+
2. bundled `panini/rules/` package data
|
|
48
|
+
3. repo path (`packages/rules-engine/rules/`) for dev checkouts
|
|
49
|
+
|
|
50
|
+
## Conformance
|
|
51
|
+
|
|
52
|
+
```
|
|
53
|
+
python3 -m unittest discover tests
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
runs every fixture in `packages/rules-engine/rules/corpus/*.json` through
|
|
57
|
+
the Python engine and asserts the same `expect` fields the TypeScript
|
|
58
|
+
corpus gate checks — channel, type, ref_no, counterparty, vpa,
|
|
59
|
+
merchant/vpa/IFSC enrichment, is_salary, date, category.
|
|
60
|
+
|
|
61
|
+
## Status
|
|
62
|
+
|
|
63
|
+
Alpha scaffold: narration pipeline (tokenizer → grammar match →
|
|
64
|
+
normalize → enrich) is at full corpus parity (76/76) and grammar data
|
|
65
|
+
is bundled for PyPI distribution. SMS `media` routing and
|
|
66
|
+
`normalizePayload` land next.
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# panini-py — Python port of Panini
|
|
2
|
+
|
|
3
|
+
Deterministic, privacy-preserving parser for Indian bank narrations.
|
|
4
|
+
This package is the Python port of `@paniniapi/rules-engine` and shares the
|
|
5
|
+
**same grammar JSONs and conformance corpus** — both runtimes are tested
|
|
6
|
+
against the identical fixture set, so parity is enforced, not assumed.
|
|
7
|
+
|
|
8
|
+
## Quick start
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
from panini import parse_narration
|
|
12
|
+
|
|
13
|
+
txn = parse_narration("UPI/20261015/DR/826384910234/SWIGGY@okaxis/Cooking/")
|
|
14
|
+
# {
|
|
15
|
+
# "channel": "UPI", "type": "DEBIT", "ref_no": "826384910234",
|
|
16
|
+
# "merchant": "Swiggy", "vpa": "okaxis", "vpa_app": "Google Pay",
|
|
17
|
+
# "vpa_bank": "Axis Bank", "category": "Food & Dining", ...
|
|
18
|
+
# }
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Where the grammars live
|
|
22
|
+
|
|
23
|
+
The grammar JSONs ship **inside the wheel** under `panini/rules/` — a
|
|
24
|
+
pip install works standalone with no repo checkout. In a development
|
|
25
|
+
checkout the engine falls back to `packages/rules-engine/rules/`; the
|
|
26
|
+
bundled copy is kept byte-identical and drift is enforced by
|
|
27
|
+
`tests/test_bundled_rules.py`. Resolution order:
|
|
28
|
+
|
|
29
|
+
1. `PANINI_RULES_DIR` env var or `rules_dir=Path(...)` argument
|
|
30
|
+
2. bundled `panini/rules/` package data
|
|
31
|
+
3. repo path (`packages/rules-engine/rules/`) for dev checkouts
|
|
32
|
+
|
|
33
|
+
## Conformance
|
|
34
|
+
|
|
35
|
+
```
|
|
36
|
+
python3 -m unittest discover tests
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
runs every fixture in `packages/rules-engine/rules/corpus/*.json` through
|
|
40
|
+
the Python engine and asserts the same `expect` fields the TypeScript
|
|
41
|
+
corpus gate checks — channel, type, ref_no, counterparty, vpa,
|
|
42
|
+
merchant/vpa/IFSC enrichment, is_salary, date, category.
|
|
43
|
+
|
|
44
|
+
## Status
|
|
45
|
+
|
|
46
|
+
Alpha scaffold: narration pipeline (tokenizer → grammar match →
|
|
47
|
+
normalize → enrich) is at full corpus parity (76/76) and grammar data
|
|
48
|
+
is bundled for PyPI distribution. SMS `media` routing and
|
|
49
|
+
`normalizePayload` land next.
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Copyright 2026 Kapil — Project Panini Contributors
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Panini — deterministic Indian bank-narration parsing. Python port of
|
|
4
|
+
@panini/rules-engine, sharing grammar JSONs and the conformance corpus."""
|
|
5
|
+
|
|
6
|
+
from .engine import Normalizer, parse_narration # noqa: F401
|
|
7
|
+
from .dates import parse_indian_date, is_plausible_date # noqa: F401
|
|
8
|
+
from .tokenizer import tokenize # noqa: F401
|
|
9
|
+
|
|
10
|
+
__version__ = "0.1.0"
|
|
11
|
+
__all__ = ["Normalizer", "parse_narration", "parse_indian_date",
|
|
12
|
+
"is_plausible_date", "tokenize"]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Copyright 2026 Kapil — Project Panini Contributors
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Indian date-shape parsing — mirrors @panini/core dates.ts."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import re
|
|
8
|
+
from datetime import date
|
|
9
|
+
|
|
10
|
+
MIN_YEAR = 1990
|
|
11
|
+
MAX_YEAR = 2100
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def _iso(year: int, month: int, day: int) -> str | None:
|
|
15
|
+
if not (MIN_YEAR <= year <= MAX_YEAR and 1 <= month <= 12 and 1 <= day <= 31):
|
|
16
|
+
return None
|
|
17
|
+
try:
|
|
18
|
+
return date(year, month, day).isoformat()
|
|
19
|
+
except ValueError:
|
|
20
|
+
return None
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _yy(yy: int) -> int:
|
|
24
|
+
return 1900 + yy if yy >= 90 else 2000 + yy
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
_ATTEMPTS = [
|
|
28
|
+
(re.compile(r"^(\d{4})(\d{2})(\d{2})$"), lambda m: (int(m[1]), int(m[2]), int(m[3]))),
|
|
29
|
+
(re.compile(r"^(\d{2})(\d{2})(\d{4})$"), lambda m: (int(m[3]), int(m[2]), int(m[1]))),
|
|
30
|
+
(re.compile(r"^(\d{2})(\d{2})(\d{2})$"), lambda m: (_yy(int(m[3])), int(m[2]), int(m[1]))),
|
|
31
|
+
(re.compile(r"^(\d{4})[-/.](\d{2})[-/.](\d{2})$"), lambda m: (int(m[1]), int(m[2]), int(m[3]))),
|
|
32
|
+
(re.compile(r"^(\d{2})[-/.](\d{2})[-/.](\d{4})$"), lambda m: (int(m[3]), int(m[2]), int(m[1]))),
|
|
33
|
+
(re.compile(r"^(\d{2})[-/.](\d{2})[-/.](\d{2})$"), lambda m: (_yy(int(m[3])), int(m[2]), int(m[1]))),
|
|
34
|
+
]
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def parse_indian_date(value: str) -> str | None:
|
|
38
|
+
"""YYYYMMDD / DDMMYYYY / DDMMYY / YYYY-MM-DD / DD-MM-YY[YY] → ISO date."""
|
|
39
|
+
v = value.strip()
|
|
40
|
+
for pattern, extract in _ATTEMPTS:
|
|
41
|
+
m = pattern.match(v)
|
|
42
|
+
if m:
|
|
43
|
+
iso = _iso(*extract(m))
|
|
44
|
+
if iso is not None:
|
|
45
|
+
return iso
|
|
46
|
+
return None
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def is_plausible_date(value: str) -> bool:
|
|
50
|
+
return parse_indian_date(value) is not None
|
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
# Copyright 2026 Kapil — Project Panini Contributors
|
|
2
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
3
|
+
"""Deterministic narration engine — Python port of @panini/core Normalizer.
|
|
4
|
+
|
|
5
|
+
Shares the same grammar JSONs and corpus fixtures as the TypeScript
|
|
6
|
+
packages; the two runtimes are conformance-tested against each other.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
import os
|
|
13
|
+
import re
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, Optional
|
|
17
|
+
|
|
18
|
+
from .dates import is_plausible_date, parse_indian_date
|
|
19
|
+
from .tokenizer import CHANNEL_KEYWORDS, DIRECTION_KEYWORDS, tokenize
|
|
20
|
+
|
|
21
|
+
# ── regex translation (JS → Python) ─────────────────────────────────────────
|
|
22
|
+
|
|
23
|
+
def translate_flags(flags: str) -> int:
|
|
24
|
+
out = 0
|
|
25
|
+
for f in flags:
|
|
26
|
+
if f == "i":
|
|
27
|
+
out |= re.IGNORECASE
|
|
28
|
+
elif f == "m":
|
|
29
|
+
out |= re.MULTILINE
|
|
30
|
+
elif f == "s":
|
|
31
|
+
out |= re.DOTALL
|
|
32
|
+
return out
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _translate(js_regex: str, flags: str = "") -> re.Pattern[str]:
|
|
36
|
+
"""JS named groups `(?<name>…)` → Python `(?P<name>…)`. Lookbehinds
|
|
37
|
+
`(?<=`/`(?<!` and non-capturing `(?:` are already valid Python."""
|
|
38
|
+
py = re.sub(r"\(\?<([A-Za-z])", r"(?P<\1", js_regex)
|
|
39
|
+
return re.compile(py, translate_flags(flags))
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# ── data model ───────────────────────────────────────────────────────────────
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True)
|
|
45
|
+
class Pattern:
|
|
46
|
+
name: str
|
|
47
|
+
rule_set_id: str
|
|
48
|
+
bank: str
|
|
49
|
+
regex: re.Pattern[str]
|
|
50
|
+
media: str
|
|
51
|
+
channel: Optional[str]
|
|
52
|
+
type: Optional[str]
|
|
53
|
+
category: Optional[str]
|
|
54
|
+
priority: int
|
|
55
|
+
confidence: float
|
|
56
|
+
flags: dict[str, str]
|
|
57
|
+
set_metadata: dict[str, str]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass(frozen=True)
|
|
61
|
+
class NormalizedTransaction:
|
|
62
|
+
channel: str
|
|
63
|
+
counterparty: Optional[str]
|
|
64
|
+
transaction_id: Optional[str]
|
|
65
|
+
vpa_handle: Optional[str]
|
|
66
|
+
type: str
|
|
67
|
+
category: str
|
|
68
|
+
date: Optional[str]
|
|
69
|
+
amount_paise: Optional[int]
|
|
70
|
+
balance_paise: Optional[int]
|
|
71
|
+
currency: str
|
|
72
|
+
account_mask: Optional[str]
|
|
73
|
+
raw: str
|
|
74
|
+
matched_rule: Optional[str]
|
|
75
|
+
bank: Optional[str]
|
|
76
|
+
confidence: float
|
|
77
|
+
metadata: dict[str, str] = field(default_factory=dict)
|
|
78
|
+
warnings: list[str] = field(default_factory=list)
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
# ── casing ───────────────────────────────────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
_CORPORATE_SUFFIXES = {"LTD", "PVT", "INC", "CORP", "CO"}
|
|
84
|
+
_ACRONYM_RE = re.compile(r"^[A-Z]{2,5}$")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _title(word: str) -> str:
|
|
88
|
+
return word[0].upper() + word[1:].lower() if word else word
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def preserve_acronym_case(value: str) -> str:
|
|
92
|
+
words = [w for w in value.strip().split() if w]
|
|
93
|
+
if len(words) <= 1:
|
|
94
|
+
word = words[0] if words else value
|
|
95
|
+
return word if re.fullmatch(r"[A-Z]{2,4}", word) else _title(word)
|
|
96
|
+
out = []
|
|
97
|
+
for w in words:
|
|
98
|
+
if w.upper() in _CORPORATE_SUFFIXES:
|
|
99
|
+
out.append(_title(w))
|
|
100
|
+
elif _ACRONYM_RE.match(w):
|
|
101
|
+
out.append(w)
|
|
102
|
+
else:
|
|
103
|
+
out.append(_title(w))
|
|
104
|
+
return " ".join(out)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ── helpers ──────────────────────────────────────────────────────────────────
|
|
108
|
+
|
|
109
|
+
_CONTROL_CHARS = re.compile(r"[\x00-\x1f\x7f]")
|
|
110
|
+
_SMS_SIGNAL = re.compile(r"\b(debited|credited|avl\s*bal|a/c\s*(?:no|xx|\*+))", re.IGNORECASE)
|
|
111
|
+
_CURRENCY_PREFIX = re.compile(r"^(?:rs\.?|inr|₹)", re.IGNORECASE)
|
|
112
|
+
_AMOUNT_BODY = re.compile(r"^\d+(?:\.\d{1,2})?$")
|
|
113
|
+
|
|
114
|
+
_DIRECTION_TYPES = {"DEBIT", "CREDIT"}
|
|
115
|
+
_REVERSAL_TYPES = {"REVERSAL", "REFUND"}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _sanitize(text: str) -> str:
|
|
119
|
+
return re.sub(r"\s+", " ", _CONTROL_CHARS.sub(" ", text)).strip()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def _detect_source(text: str) -> str:
|
|
123
|
+
return "sms" if _SMS_SIGNAL.search(text) else "narration"
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def _parse_paise(value: str) -> Optional[int]:
|
|
127
|
+
stripped = _CURRENCY_PREFIX.sub("", value.strip()).replace(",", "")
|
|
128
|
+
if not _AMOUNT_BODY.match(stripped):
|
|
129
|
+
return None
|
|
130
|
+
return round(float(stripped) * 100)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _clean_counterparty(value: Optional[str]) -> Optional[str]:
|
|
134
|
+
if not value:
|
|
135
|
+
return None
|
|
136
|
+
cleaned = re.sub(
|
|
137
|
+
r"^(?:(?:M/S|TO|BY|FROM|AT|THE|UPI|IMPS|NEFT|RTGS|ECS|ACH|NACH)[\s\-/]+)+",
|
|
138
|
+
"", value, flags=re.IGNORECASE,
|
|
139
|
+
)
|
|
140
|
+
cleaned = re.sub(r"[\s\-./]+$", "", cleaned)
|
|
141
|
+
cleaned = re.sub(r"\s+", " ", cleaned).strip()
|
|
142
|
+
return preserve_acronym_case(cleaned) if cleaned else None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def _resolve_type(pattern_type, direction_group, flag_types) -> str:
|
|
146
|
+
if pattern_type:
|
|
147
|
+
return pattern_type
|
|
148
|
+
if direction_group:
|
|
149
|
+
mapped = DIRECTION_KEYWORDS.get(direction_group.upper())
|
|
150
|
+
if mapped:
|
|
151
|
+
return mapped
|
|
152
|
+
for t in flag_types:
|
|
153
|
+
if t in _DIRECTION_TYPES:
|
|
154
|
+
return t
|
|
155
|
+
for t in flag_types:
|
|
156
|
+
if t in _REVERSAL_TYPES:
|
|
157
|
+
return t
|
|
158
|
+
return "UNKNOWN"
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
# ── grammar compilation ──────────────────────────────────────────────────────
|
|
162
|
+
|
|
163
|
+
def compile_rule_sets(rule_sets: list[dict[str, Any]]) -> list[Pattern]:
|
|
164
|
+
"""Ordering: priority desc, bank-specific before GENERIC, stable."""
|
|
165
|
+
compiled: list[Pattern] = []
|
|
166
|
+
for rs in rule_sets:
|
|
167
|
+
for p in rs.get("patterns", []):
|
|
168
|
+
compiled.append(Pattern(
|
|
169
|
+
name=p["name"],
|
|
170
|
+
rule_set_id=rs["id"],
|
|
171
|
+
bank=rs["bank"],
|
|
172
|
+
regex=_translate(p["regex"], p.get("regexFlags", "")),
|
|
173
|
+
media=p.get("media", "any"),
|
|
174
|
+
channel=p.get("channel"),
|
|
175
|
+
type=p.get("type"),
|
|
176
|
+
category=p.get("category"),
|
|
177
|
+
priority=p.get("priority", 0),
|
|
178
|
+
confidence=p.get("confidence", 0.9),
|
|
179
|
+
flags=p.get("flags", {}),
|
|
180
|
+
set_metadata=p.get("setMetadata", {}),
|
|
181
|
+
))
|
|
182
|
+
return sorted(
|
|
183
|
+
compiled,
|
|
184
|
+
key=lambda p: (-p.priority, p.bank == "GENERIC"),
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def load_builtin_rules(rules_dir: Path) -> tuple[list[dict], dict[str, str]]:
|
|
189
|
+
grammars, corpus = [], {}
|
|
190
|
+
for path in sorted(rules_dir.glob("*.json")):
|
|
191
|
+
if path.name == "categories.json":
|
|
192
|
+
corpus = json.loads(path.read_text())
|
|
193
|
+
continue
|
|
194
|
+
grammars.append(json.loads(path.read_text()))
|
|
195
|
+
return grammars, corpus
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
# ── normalizer ───────────────────────────────────────────────────────────────
|
|
199
|
+
|
|
200
|
+
class Normalizer:
|
|
201
|
+
def __init__(self, rule_sets: list[dict], categories: dict[str, str]):
|
|
202
|
+
self.patterns = compile_rule_sets(rule_sets)
|
|
203
|
+
merged: dict[str, str] = {}
|
|
204
|
+
for rs in rule_sets:
|
|
205
|
+
merged.update(rs.get("categories", {}))
|
|
206
|
+
merged.update(categories)
|
|
207
|
+
self.category_rules = [
|
|
208
|
+
(kw, cat, re.compile(r"\b" + re.escape(kw) + r"\b", re.IGNORECASE))
|
|
209
|
+
for kw, cat in sorted(merged.items(), key=lambda kv: -len(kv[0]))
|
|
210
|
+
]
|
|
211
|
+
|
|
212
|
+
def _candidates(self, source: str, bank_hint: Optional[str]) -> list[Pattern]:
|
|
213
|
+
applicable = [p for p in self.patterns if p.media in ("any", source)]
|
|
214
|
+
if not bank_hint:
|
|
215
|
+
return applicable
|
|
216
|
+
hint = bank_hint.upper()
|
|
217
|
+
preferred = [p for p in applicable if p.bank.upper() == hint]
|
|
218
|
+
rest = [p for p in applicable if p.bank.upper() != hint]
|
|
219
|
+
return preferred + rest
|
|
220
|
+
|
|
221
|
+
def _channel_from_tokens(self, tokens) -> Optional[str]:
|
|
222
|
+
for t in tokens:
|
|
223
|
+
ch = CHANNEL_KEYWORDS.get(t.value.upper())
|
|
224
|
+
if ch:
|
|
225
|
+
return ch
|
|
226
|
+
return None
|
|
227
|
+
|
|
228
|
+
def _categorize(self, text: str) -> Optional[str]:
|
|
229
|
+
for _, cat, matcher in self.category_rules:
|
|
230
|
+
if matcher.search(text):
|
|
231
|
+
return cat
|
|
232
|
+
return None
|
|
233
|
+
|
|
234
|
+
def _empty(self, raw: str, warnings: list[str]) -> NormalizedTransaction:
|
|
235
|
+
return NormalizedTransaction(
|
|
236
|
+
channel="UNKNOWN", counterparty=None, transaction_id=None,
|
|
237
|
+
vpa_handle=None, type="UNKNOWN", category="Uncategorized",
|
|
238
|
+
date=None, amount_paise=None, balance_paise=None,
|
|
239
|
+
currency="INR", account_mask=None, raw=raw,
|
|
240
|
+
matched_rule=None, bank=None, confidence=0.0,
|
|
241
|
+
warnings=list(warnings),
|
|
242
|
+
)
|
|
243
|
+
|
|
244
|
+
def _from_match(self, pattern: Pattern, m: re.Match, text: str,
|
|
245
|
+
raw: str, warnings: list[str]) -> NormalizedTransaction:
|
|
246
|
+
groups = {k: v for k, v in (m.groupdict() or {}).items()}
|
|
247
|
+
metadata: dict[str, str] = {}
|
|
248
|
+
|
|
249
|
+
flag_types, applied = [], []
|
|
250
|
+
if groups.get("tail"):
|
|
251
|
+
for tok in tokenize(groups["tail"], is_plausible_date):
|
|
252
|
+
upper = tok.value.upper()
|
|
253
|
+
mapped = pattern.flags.get(upper) or DIRECTION_KEYWORDS.get(upper)
|
|
254
|
+
if mapped:
|
|
255
|
+
flag_types.append(mapped)
|
|
256
|
+
applied.append(upper)
|
|
257
|
+
if applied:
|
|
258
|
+
metadata["flags"] = ",".join(applied)
|
|
259
|
+
if groups.get("remarks"):
|
|
260
|
+
metadata["remarks"] = groups["remarks"].strip()
|
|
261
|
+
if groups.get("bankCode"):
|
|
262
|
+
metadata["bank_code"] = groups["bankCode"].strip()
|
|
263
|
+
metadata.update(pattern.set_metadata)
|
|
264
|
+
|
|
265
|
+
tokens = tokenize(text, is_plausible_date)
|
|
266
|
+
channel = pattern.channel or self._channel_from_tokens(tokens) or "UNKNOWN"
|
|
267
|
+
txn_type = _resolve_type(pattern.type, groups.get("direction"), flag_types)
|
|
268
|
+
|
|
269
|
+
counterparty = _clean_counterparty(groups.get("counterparty"))
|
|
270
|
+
txn_id = (groups.get("txnId") or "").strip() or None
|
|
271
|
+
|
|
272
|
+
vpa = groups.get("vpaHandle")
|
|
273
|
+
if vpa is not None:
|
|
274
|
+
vpa = vpa.lstrip("@").lower()
|
|
275
|
+
|
|
276
|
+
date_val = self._parse_field(groups.get("date"), parse_indian_date, "date", warnings)
|
|
277
|
+
amount = self._parse_field(groups.get("amount"), _parse_paise, "amount", warnings)
|
|
278
|
+
balance = self._parse_field(groups.get("balance"), _parse_paise, "balance", warnings)
|
|
279
|
+
account_mask = (groups.get("accountMask") or "").strip() or None
|
|
280
|
+
|
|
281
|
+
category = (
|
|
282
|
+
pattern.category
|
|
283
|
+
or self._categorize(f"{counterparty or ''} {text}")
|
|
284
|
+
or "Uncategorized"
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
return NormalizedTransaction(
|
|
288
|
+
channel=channel, counterparty=counterparty, transaction_id=txn_id,
|
|
289
|
+
vpa_handle=vpa, type=txn_type, category=category, date=date_val,
|
|
290
|
+
amount_paise=amount, balance_paise=balance, currency="INR",
|
|
291
|
+
account_mask=account_mask, raw=raw,
|
|
292
|
+
matched_rule=f"{pattern.rule_set_id}:{pattern.name}",
|
|
293
|
+
bank=None if pattern.bank == "GENERIC" else pattern.bank,
|
|
294
|
+
confidence=pattern.confidence, metadata=metadata,
|
|
295
|
+
warnings=list(warnings),
|
|
296
|
+
)
|
|
297
|
+
|
|
298
|
+
@staticmethod
|
|
299
|
+
def _parse_field(value, parser, label, warnings):
|
|
300
|
+
if value is None or not value.strip():
|
|
301
|
+
return None
|
|
302
|
+
parsed = parser(value.strip())
|
|
303
|
+
if parsed is None:
|
|
304
|
+
warnings.append(f"unparseable {label} '{value.strip()}'")
|
|
305
|
+
return parsed
|
|
306
|
+
|
|
307
|
+
def _heuristic(self, text: str, raw: str, bank_hint, warnings) -> NormalizedTransaction:
|
|
308
|
+
tokens = tokenize(text, is_plausible_date)
|
|
309
|
+
metadata: dict[str, str] = {}
|
|
310
|
+
flag_types, applied = [], []
|
|
311
|
+
for t in tokens:
|
|
312
|
+
mapped = DIRECTION_KEYWORDS.get(t.value.upper())
|
|
313
|
+
if mapped:
|
|
314
|
+
flag_types.append(mapped)
|
|
315
|
+
applied.append(t.value.upper())
|
|
316
|
+
if applied:
|
|
317
|
+
metadata["flags"] = ",".join(applied)
|
|
318
|
+
|
|
319
|
+
channel = self._channel_from_tokens(tokens) or "UNKNOWN"
|
|
320
|
+
txn_type = _resolve_type(None, None, flag_types)
|
|
321
|
+
|
|
322
|
+
txn_id, vpa = None, None
|
|
323
|
+
vpa_tok = next((t for t in tokens if t.kind == "VPA"), None)
|
|
324
|
+
if vpa_tok:
|
|
325
|
+
left, _, right = vpa_tok.value.partition("@")
|
|
326
|
+
if left.isdigit():
|
|
327
|
+
txn_id = left
|
|
328
|
+
if re.fullmatch(r"[a-zA-Z0-9]+", right or ""):
|
|
329
|
+
vpa = right.lower()
|
|
330
|
+
if txn_id is None:
|
|
331
|
+
nums = sorted(
|
|
332
|
+
(t for t in tokens if t.kind == "NUMERIC" and len(t.value) >= 6),
|
|
333
|
+
key=lambda t: -len(t.value),
|
|
334
|
+
)
|
|
335
|
+
txn_id = nums[0].value if nums else None
|
|
336
|
+
|
|
337
|
+
date_tok = next((t for t in tokens if t.kind == "DATE"), None)
|
|
338
|
+
date_val = parse_indian_date(date_tok.value) if date_tok else None
|
|
339
|
+
amt_tok = next((t for t in tokens if t.kind == "AMOUNT"), None)
|
|
340
|
+
amount = _parse_paise(amt_tok.value) if amt_tok else None
|
|
341
|
+
|
|
342
|
+
channel_pos = next(
|
|
343
|
+
(t.position for t in tokens if t.value.upper() in CHANNEL_KEYWORDS), None
|
|
344
|
+
)
|
|
345
|
+
cp_tok = next(
|
|
346
|
+
(t for t in tokens
|
|
347
|
+
if t.kind in ("ALPHA", "ALNUM") and len(t.value) >= 3
|
|
348
|
+
and t.value.upper() not in CHANNEL_KEYWORDS
|
|
349
|
+
and t.value.upper() not in DIRECTION_KEYWORDS
|
|
350
|
+
and (channel_pos is None or t.position > channel_pos)),
|
|
351
|
+
None,
|
|
352
|
+
)
|
|
353
|
+
counterparty = preserve_acronym_case(cp_tok.value) if cp_tok else None
|
|
354
|
+
|
|
355
|
+
category = self._categorize(f"{counterparty or ''} {text}") or "Uncategorized"
|
|
356
|
+
warnings.append("no grammar pattern matched; applied heuristic tokenization")
|
|
357
|
+
|
|
358
|
+
return NormalizedTransaction(
|
|
359
|
+
channel=channel, counterparty=counterparty, transaction_id=txn_id,
|
|
360
|
+
vpa_handle=vpa, type=txn_type, category=category, date=date_val,
|
|
361
|
+
amount_paise=amount, balance_paise=None, currency="INR",
|
|
362
|
+
account_mask=None, raw=raw, matched_rule=None,
|
|
363
|
+
bank=bank_hint, confidence=0.5, metadata=metadata,
|
|
364
|
+
warnings=list(warnings),
|
|
365
|
+
)
|
|
366
|
+
|
|
367
|
+
def normalize(self, input_: str | dict) -> NormalizedTransaction:
|
|
368
|
+
req = {"text": input_} if isinstance(input_, str) else dict(input_)
|
|
369
|
+
raw = req.get("text", "")
|
|
370
|
+
text = _sanitize(raw)
|
|
371
|
+
if not text:
|
|
372
|
+
return self._empty(raw, ["input is empty"])
|
|
373
|
+
source = req.get("source")
|
|
374
|
+
if source in (None, "auto"):
|
|
375
|
+
source = _detect_source(text)
|
|
376
|
+
bank_hint = req.get("bankHint")
|
|
377
|
+
warnings: list[str] = []
|
|
378
|
+
for pattern in self._candidates(source, bank_hint):
|
|
379
|
+
m = pattern.regex.search(text)
|
|
380
|
+
if m:
|
|
381
|
+
return self._from_match(pattern, m, text, raw, warnings)
|
|
382
|
+
return self._heuristic(text, raw, bank_hint, warnings)
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
# ── dictionaries + facade ────────────────────────────────────────────────────
|
|
386
|
+
|
|
387
|
+
def _default_rules_dir() -> Path:
|
|
388
|
+
"""Resolution order: PANINI_RULES_DIR env → bundled package data →
|
|
389
|
+
repo layout (development checkout)."""
|
|
390
|
+
env = os.environ.get("PANINI_RULES_DIR")
|
|
391
|
+
if env:
|
|
392
|
+
return Path(env)
|
|
393
|
+
bundled = Path(__file__).resolve().parent / "rules"
|
|
394
|
+
if (bundled / "generic.json").exists():
|
|
395
|
+
return bundled
|
|
396
|
+
return Path(__file__).resolve().parents[2] / "packages" / "rules-engine" / "rules"
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
_RULES_DIR = _default_rules_dir()
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
def _load_entities(rules_dir: Path) -> dict[str, Any]:
|
|
403
|
+
ent = rules_dir / "entities"
|
|
404
|
+
out: dict[str, Any] = {"merchants": [], "vpa-handles": {}, "ifsc-codes": {}}
|
|
405
|
+
for name in ("merchants", "vpa-handles", "ifsc-codes"):
|
|
406
|
+
p = ent / f"{name}.json"
|
|
407
|
+
if p.exists():
|
|
408
|
+
data = json.loads(p.read_text())
|
|
409
|
+
if name == "merchants":
|
|
410
|
+
out[name] = data.get("merchants", [])
|
|
411
|
+
elif name == "vpa-handles":
|
|
412
|
+
out[name] = data.get("handles", {})
|
|
413
|
+
else:
|
|
414
|
+
out[name] = data.get("banks", {})
|
|
415
|
+
return out
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _normalize_key(value: str) -> str:
|
|
419
|
+
return re.sub(r"\s+", " ", re.sub(r"[^A-Z0-9 ]", " ", value.upper())).strip()
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def _lookup_merchant(counterparty: Optional[str], merchants: list[dict]) -> Optional[dict]:
|
|
423
|
+
if not counterparty:
|
|
424
|
+
return None
|
|
425
|
+
key = _normalize_key(counterparty)
|
|
426
|
+
for entry in merchants:
|
|
427
|
+
for alias in [entry["canonical"], *entry.get("aliases", [])]:
|
|
428
|
+
if _normalize_key(alias) == key:
|
|
429
|
+
return entry
|
|
430
|
+
best, best_len = None, 0
|
|
431
|
+
for entry in merchants:
|
|
432
|
+
for alias in [entry["canonical"], *entry.get("aliases", [])]:
|
|
433
|
+
norm = _normalize_key(alias)
|
|
434
|
+
if len(norm) > best_len and key.startswith(norm + " "):
|
|
435
|
+
best, best_len = entry, len(norm)
|
|
436
|
+
return best
|
|
437
|
+
|
|
438
|
+
|
|
439
|
+
_IFSC_RE = re.compile(r"([A-Z]{4})0[A-Z0-9]{6}")
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def _lookup_ifsc(value: Optional[str], banks: dict) -> Optional[dict]:
|
|
443
|
+
if not value:
|
|
444
|
+
return None
|
|
445
|
+
upper = value.upper()
|
|
446
|
+
m = _IFSC_RE.search(upper)
|
|
447
|
+
if m:
|
|
448
|
+
prefix = m.group(1)
|
|
449
|
+
elif re.fullmatch(r"[A-Z]{4}", upper.strip()):
|
|
450
|
+
prefix = upper.strip()
|
|
451
|
+
else:
|
|
452
|
+
return None
|
|
453
|
+
return banks.get(prefix)
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def parse_narration(input_: str | dict, *, enrich: bool = True,
|
|
457
|
+
rules_dir: Optional[Path] = None) -> dict[str, Any]:
|
|
458
|
+
"""Parse a bank narration into the flattened ParsedBankNarration shape —
|
|
459
|
+
the same output contract as parseBankNarration() in TypeScript."""
|
|
460
|
+
rd = rules_dir or _RULES_DIR
|
|
461
|
+
rule_sets, categories = load_builtin_rules(rd)
|
|
462
|
+
norm = Normalizer(rule_sets, categories)
|
|
463
|
+
txn = norm.normalize(input_)
|
|
464
|
+
|
|
465
|
+
merchant_entry = vpa_info = ifsc_info = None
|
|
466
|
+
if enrich:
|
|
467
|
+
entities = _load_entities(rd)
|
|
468
|
+
merchant_entry = _lookup_merchant(txn.counterparty, entities["merchants"])
|
|
469
|
+
if txn.vpa_handle:
|
|
470
|
+
vpa_info = entities["vpa-handles"].get(txn.vpa_handle)
|
|
471
|
+
ifsc_src = txn.metadata.get("bank_code") or txn.raw
|
|
472
|
+
ifsc_info = _lookup_ifsc(ifsc_src, entities["ifsc-codes"])
|
|
473
|
+
|
|
474
|
+
merchant = (merchant_entry or {}).get("canonical") or txn.counterparty
|
|
475
|
+
category = (
|
|
476
|
+
merchant_entry["category"]
|
|
477
|
+
if txn.category == "Uncategorized" and merchant_entry
|
|
478
|
+
else txn.category
|
|
479
|
+
)
|
|
480
|
+
return {
|
|
481
|
+
"channel": txn.channel,
|
|
482
|
+
"type": txn.type,
|
|
483
|
+
"ref_no": txn.transaction_id,
|
|
484
|
+
"merchant": merchant,
|
|
485
|
+
"counterparty": txn.counterparty,
|
|
486
|
+
"vpa": txn.vpa_handle,
|
|
487
|
+
"is_salary": txn.metadata.get("is_salary") == "true",
|
|
488
|
+
"date": txn.date,
|
|
489
|
+
"category": category,
|
|
490
|
+
"confidence": txn.confidence,
|
|
491
|
+
"matched_rule": txn.matched_rule,
|
|
492
|
+
"warnings": txn.warnings,
|
|
493
|
+
"merchant_canonical": (merchant_entry or {}).get("canonical"),
|
|
494
|
+
"vpa_app": (vpa_info or {}).get("app"),
|
|
495
|
+
"vpa_bank": (vpa_info or {}).get("bank"),
|
|
496
|
+
"ifsc_bank": (ifsc_info or {}).get("name"),
|
|
497
|
+
}
|