sheet-unmess 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sheet_unmess/__init__.py +16 -0
- sheet_unmess/ai_rules.py +79 -0
- sheet_unmess/classifier.py +94 -0
- sheet_unmess/cli.py +98 -0
- sheet_unmess/core.py +180 -0
- sheet_unmess/detector.py +164 -0
- sheet_unmess/mcp_server.py +154 -0
- sheet_unmess/normalizer.py +125 -0
- sheet_unmess/receipt.py +121 -0
- sheet_unmess/reconciler.py +59 -0
- sheet_unmess-1.0.0.dist-info/METADATA +139 -0
- sheet_unmess-1.0.0.dist-info/RECORD +14 -0
- sheet_unmess-1.0.0.dist-info/WHEEL +4 -0
- sheet_unmess-1.0.0.dist-info/entry_points.txt +3 -0
sheet_unmess/__init__.py
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""
|
|
2
|
+
sheet-unmess: Forensic Messy-Excel & Dirty-CSV Ingestor + Audit Receipt.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from sheet_unmess.core import read_messy, unmess
|
|
6
|
+
from sheet_unmess.receipt import AuditReceipt, CleaningReport
|
|
7
|
+
from sheet_unmess.ai_rules import install_ai_rules
|
|
8
|
+
|
|
9
|
+
__version__ = "1.0.0"
|
|
10
|
+
__all__ = [
|
|
11
|
+
"read_messy",
|
|
12
|
+
"unmess",
|
|
13
|
+
"AuditReceipt",
|
|
14
|
+
"CleaningReport",
|
|
15
|
+
"install_ai_rules",
|
|
16
|
+
]
|
sheet_unmess/ai_rules.py
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""AI editor rules and MCP configuration installer."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import os
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
|
|
8
|
+
CURSOR_RULE_CONTENT = """# Spreadsheet ingestion rules
|
|
9
|
+
|
|
10
|
+
When working with Excel (.xlsx) or CSV files, prefer `sheet_unmess.read_messy`
|
|
11
|
+
over raw pandas readers when the file may contain merged/multi-row headers,
|
|
12
|
+
embedded subtotals, mixed locale-formatted numbers, or title/footnote rows.
|
|
13
|
+
|
|
14
|
+
```python
|
|
15
|
+
from sheet_unmess import read_messy
|
|
16
|
+
|
|
17
|
+
report = read_messy("report.xlsx", return_report=True)
|
|
18
|
+
df = report.data
|
|
19
|
+
print(report.receipt.to_markdown())
|
|
20
|
+
```
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
def _merge_json(path: Path, key: str, value: dict) -> None:
|
|
24
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
25
|
+
current = {}
|
|
26
|
+
if path.exists():
|
|
27
|
+
try:
|
|
28
|
+
current = json.loads(path.read_text(encoding="utf-8"))
|
|
29
|
+
except (json.JSONDecodeError, OSError):
|
|
30
|
+
current = {}
|
|
31
|
+
current.setdefault(key, {}).update(value)
|
|
32
|
+
path.write_text(json.dumps(current, indent=2), encoding="utf-8")
|
|
33
|
+
|
|
34
|
+
def install_ai_rules(project_root: str = ".") -> dict[str, bool]:
|
|
35
|
+
"""Install project-local rules and Cursor MCP config; preserve other MCP entries."""
|
|
36
|
+
root = Path(project_root).resolve()
|
|
37
|
+
results: dict[str, bool] = {}
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
(root / ".cursorrules").write_text(CURSOR_RULE_CONTENT, encoding="utf-8")
|
|
41
|
+
results[".cursorrules"] = True
|
|
42
|
+
except OSError:
|
|
43
|
+
results[".cursorrules"] = False
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
(root / ".windsurfrules").write_text(CURSOR_RULE_CONTENT, encoding="utf-8")
|
|
47
|
+
results[".windsurfrules"] = True
|
|
48
|
+
except OSError:
|
|
49
|
+
results[".windsurfrules"] = False
|
|
50
|
+
|
|
51
|
+
try:
|
|
52
|
+
_merge_json(
|
|
53
|
+
root / ".cursor" / "mcp.json",
|
|
54
|
+
"mcpServers",
|
|
55
|
+
{"sheet-unmess": {"command": "python", "args": ["-m", "sheet_unmess.mcp_server"]}},
|
|
56
|
+
)
|
|
57
|
+
results[".cursor/mcp.json"] = True
|
|
58
|
+
except (OSError, TypeError):
|
|
59
|
+
results[".cursor/mcp.json"] = False
|
|
60
|
+
|
|
61
|
+
# Claude Desktop locations differ by operating system.
|
|
62
|
+
home = Path.home()
|
|
63
|
+
if os.name == "nt":
|
|
64
|
+
claude_path = home / "AppData" / "Roaming" / "Claude" / "claude_desktop_config.json"
|
|
65
|
+
elif os.uname().sysname == "Darwin":
|
|
66
|
+
claude_path = home / "Library" / "Application Support" / "Claude" / "claude_desktop_config.json"
|
|
67
|
+
else:
|
|
68
|
+
claude_path = home / ".config" / "Claude" / "claude_desktop_config.json"
|
|
69
|
+
|
|
70
|
+
try:
|
|
71
|
+
_merge_json(
|
|
72
|
+
claude_path,
|
|
73
|
+
"mcpServers",
|
|
74
|
+
{"sheet-unmess": {"command": "python", "args": ["-m", "sheet_unmess.mcp_server"]}},
|
|
75
|
+
)
|
|
76
|
+
results["claude_desktop"] = True
|
|
77
|
+
except (OSError, TypeError):
|
|
78
|
+
results["claude_desktop"] = False
|
|
79
|
+
return results
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Row Classification Engine.
|
|
3
|
+
Forensically categorizes every row into:
|
|
4
|
+
- TITLE_BANNER
|
|
5
|
+
- DATA
|
|
6
|
+
- SUBTOTAL
|
|
7
|
+
- GRAND_TOTAL
|
|
8
|
+
- FOOTNOTE
|
|
9
|
+
- BLANK_PADDING
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
import re
|
|
14
|
+
from typing import Any, List, Optional
|
|
15
|
+
from sheet_unmess.normalizer import parse_forensic_number
|
|
16
|
+
|
|
17
|
+
RE_GRAND_TOTAL = re.compile(
|
|
18
|
+
r"\b(grand\s*total|overall\s*total|gesamtsumme|total\s*general|solde\s*final)\b",
|
|
19
|
+
re.IGNORECASE,
|
|
20
|
+
)
|
|
21
|
+
RE_SUBTOTAL = re.compile(
|
|
22
|
+
r"\b(sub\s*total|subtotal|summe|zwischensumme|sous\s*total|totales?|total)\b",
|
|
23
|
+
re.IGNORECASE,
|
|
24
|
+
)
|
|
25
|
+
RE_FOOTNOTE = re.compile(
|
|
26
|
+
r"^\s*([*†‡§#^]|note:|notes:|source:|disclaimer:|confidential|prepared\s+by|copyright)",
|
|
27
|
+
re.IGNORECASE,
|
|
28
|
+
)
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
class RowClassifier:
|
|
32
|
+
@staticmethod
|
|
33
|
+
def is_blank(row: List[Any]) -> bool:
|
|
34
|
+
for cell in row:
|
|
35
|
+
if cell is not None and str(cell).strip() not in ("", "None", "nan"):
|
|
36
|
+
return False
|
|
37
|
+
return True
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def is_footnote(row: List[Any], total_cols: int) -> bool:
|
|
41
|
+
# A footnote typically has content in the first 1-2 columns and is empty everywhere else
|
|
42
|
+
non_empty = [c for c in row if c is not None and str(c).strip() not in ("", "None", "nan")]
|
|
43
|
+
if not non_empty:
|
|
44
|
+
return False
|
|
45
|
+
first_str = str(non_empty[0]).strip()
|
|
46
|
+
if RE_FOOTNOTE.match(first_str):
|
|
47
|
+
return True
|
|
48
|
+
# If there is only 1 non-empty cell in a wide table, and it is a long text string (>30 chars)
|
|
49
|
+
if len(non_empty) == 1 and len(first_str) > 30 and total_cols >= 3:
|
|
50
|
+
return True
|
|
51
|
+
return False
|
|
52
|
+
|
|
53
|
+
@classmethod
|
|
54
|
+
def classify_row(
|
|
55
|
+
cls,
|
|
56
|
+
row: List[Any],
|
|
57
|
+
row_idx: int,
|
|
58
|
+
header_end_idx: int,
|
|
59
|
+
total_rows: int,
|
|
60
|
+
col_names: Optional[List[str]] = None,
|
|
61
|
+
) -> str:
|
|
62
|
+
if cls.is_blank(row):
|
|
63
|
+
return "BLANK_PADDING"
|
|
64
|
+
|
|
65
|
+
# Rows prior to data block
|
|
66
|
+
if row_idx < header_end_idx:
|
|
67
|
+
return "TITLE_BANNER"
|
|
68
|
+
|
|
69
|
+
total_cols = len(row)
|
|
70
|
+
if cls.is_footnote(row, total_cols):
|
|
71
|
+
return "FOOTNOTE"
|
|
72
|
+
|
|
73
|
+
# Check for subtotal or grand total markers
|
|
74
|
+
first_cells_str = " ".join([str(c) for c in row[:3] if c is not None])
|
|
75
|
+
|
|
76
|
+
if RE_GRAND_TOTAL.search(first_cells_str):
|
|
77
|
+
return "GRAND_TOTAL"
|
|
78
|
+
|
|
79
|
+
if RE_SUBTOTAL.search(first_cells_str):
|
|
80
|
+
return "SUBTOTAL"
|
|
81
|
+
|
|
82
|
+
return "DATA"
|
|
83
|
+
|
|
84
|
+
@classmethod
|
|
85
|
+
def extract_row_numerics(cls, row: List[Any], col_names: List[str]) -> dict[str, float]:
|
|
86
|
+
"""Extracts all parsable numbers from a row keyed by column name."""
|
|
87
|
+
numerics = {}
|
|
88
|
+
for c_idx, cell in enumerate(row):
|
|
89
|
+
if c_idx < len(col_names):
|
|
90
|
+
col = col_names[c_idx]
|
|
91
|
+
parsed, is_num = parse_forensic_number(cell)
|
|
92
|
+
if is_num and parsed is not None:
|
|
93
|
+
numerics[col] = float(parsed)
|
|
94
|
+
return numerics
|
sheet_unmess/cli.py
ADDED
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Rich Command-Line Interface for sheet-unmess.
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
import argparse
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from sheet_unmess.core import read_messy
|
|
10
|
+
from sheet_unmess.ai_rules import install_ai_rules
|
|
11
|
+
from sheet_unmess.mcp_server import main as mcp_main
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def main():
|
|
15
|
+
parser = argparse.ArgumentParser(
|
|
16
|
+
prog="sheet-unmess",
|
|
17
|
+
description="Forensic Messy-Excel & Dirty-CSV Ingestor + Tamper-Evident Audit Receipt."
|
|
18
|
+
)
|
|
19
|
+
subparsers = parser.add_subparsers(dest="command")
|
|
20
|
+
|
|
21
|
+
# Command: clean
|
|
22
|
+
clean_p = subparsers.add_parser("clean", help="Clean a messy spreadsheet")
|
|
23
|
+
clean_p.add_argument("file", help="Path to .xlsx or .csv")
|
|
24
|
+
clean_p.add_argument("--sheet", default=0, help="Sheet name or index (for Excel)")
|
|
25
|
+
clean_p.add_argument("--out", "-o", help="Output path (.csv or .parquet)")
|
|
26
|
+
clean_p.add_argument("--receipt", "-r", help="Save Audit Receipt markdown file path")
|
|
27
|
+
clean_p.add_argument("--dayfirst", action="store_true", help="Parse dates EU style (DD/MM/YYYY)")
|
|
28
|
+
|
|
29
|
+
# Command: init-ai
|
|
30
|
+
subparsers.add_parser("init-ai", help="Inject MCP server and rules into Cursor, Windsurf, and Claude")
|
|
31
|
+
|
|
32
|
+
# Command: mcp
|
|
33
|
+
subparsers.add_parser("mcp", help="Launch native Stdio Model Context Protocol (MCP) server")
|
|
34
|
+
|
|
35
|
+
args = parser.parse_args()
|
|
36
|
+
|
|
37
|
+
if args.command == "mcp":
|
|
38
|
+
mcp_main()
|
|
39
|
+
return
|
|
40
|
+
|
|
41
|
+
if args.command == "init-ai":
|
|
42
|
+
res = install_ai_rules()
|
|
43
|
+
print("🚀 Successfully provisioned AI agent configurations:")
|
|
44
|
+
for target, status in res.items():
|
|
45
|
+
icon = "✅" if status else "⚠️"
|
|
46
|
+
print(f" {icon} {target}")
|
|
47
|
+
print("\nYour AI assistant (Cursor, Windsurf, Claude) is now primed to automatically clean spreadsheets with sheet-unmess!")
|
|
48
|
+
return
|
|
49
|
+
|
|
50
|
+
if args.command == "clean":
|
|
51
|
+
file_path = Path(args.file)
|
|
52
|
+
if not file_path.exists():
|
|
53
|
+
print(f"❌ File not found: {file_path}", file=sys.stderr)
|
|
54
|
+
sys.exit(1)
|
|
55
|
+
|
|
56
|
+
print(f"🔍 Forensically ingesting `{file_path}`...")
|
|
57
|
+
report = read_messy(
|
|
58
|
+
file_path,
|
|
59
|
+
sheet_name=args.sheet,
|
|
60
|
+
date_dayfirst=args.dayfirst,
|
|
61
|
+
return_report=True
|
|
62
|
+
)
|
|
63
|
+
|
|
64
|
+
df = report.data
|
|
65
|
+
receipt = report.receipt
|
|
66
|
+
|
|
67
|
+
print(f"✨ Ingestion complete: {receipt.raw_shape} ➔ {receipt.clean_shape} in {receipt.execution_time_ms:.1f}ms")
|
|
68
|
+
|
|
69
|
+
if receipt.reconciliations:
|
|
70
|
+
print("\n📊 Invariant Verification:")
|
|
71
|
+
for rec in receipt.reconciliations:
|
|
72
|
+
icon = "✅" if rec.status in ("VERIFIED_EXACT", "ROUNDING_TOLERANCE") else "🚨"
|
|
73
|
+
print(f" {icon} {rec.column}: reported {rec.reported_total:,.2f} vs sum {rec.computed_sum:,.2f} ({rec.status})")
|
|
74
|
+
|
|
75
|
+
# Save output data
|
|
76
|
+
if args.out:
|
|
77
|
+
out_p = Path(args.out)
|
|
78
|
+
if out_p.suffix == ".parquet":
|
|
79
|
+
df.to_parquet(out_p, index=False)
|
|
80
|
+
else:
|
|
81
|
+
df.to_csv(out_p, index=False)
|
|
82
|
+
print(f"\n💾 Clean data written to: {out_p}")
|
|
83
|
+
else:
|
|
84
|
+
print("\nPreview of cleaned data:")
|
|
85
|
+
print(df.head(10).to_string())
|
|
86
|
+
|
|
87
|
+
# Save receipt
|
|
88
|
+
if args.receipt:
|
|
89
|
+
rec_p = Path(args.receipt)
|
|
90
|
+
rec_p.write_text(receipt.to_markdown(), encoding="utf-8")
|
|
91
|
+
print(f"🛡️ Audit Receipt written to: {rec_p}")
|
|
92
|
+
|
|
93
|
+
else:
|
|
94
|
+
parser.print_help()
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
if __name__ == "__main__":
|
|
98
|
+
main()
|
sheet_unmess/core.py
ADDED
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Main Ingestion Engine and Public API: read_messy() and unmess().
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
import time
|
|
7
|
+
from typing import Any, Dict, List, Literal, Optional, Union
|
|
8
|
+
import pandas as pd
|
|
9
|
+
|
|
10
|
+
from sheet_unmess.detector import load_raw_grid_from_file, HeaderDetector
|
|
11
|
+
from sheet_unmess.classifier import RowClassifier
|
|
12
|
+
from sheet_unmess.normalizer import parse_forensic_number, parse_forensic_date
|
|
13
|
+
from sheet_unmess.reconciler import MathematicalReconciler
|
|
14
|
+
from sheet_unmess.receipt import (
|
|
15
|
+
AuditReceipt,
|
|
16
|
+
CleaningReport,
|
|
17
|
+
DroppedRowInfo,
|
|
18
|
+
ColumnTransformation,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def read_messy(
|
|
23
|
+
filepath_or_buffer: Any,
|
|
24
|
+
sheet_name: Union[str, int] = 0,
|
|
25
|
+
return_report: bool = False,
|
|
26
|
+
engine: Literal["pandas", "polars"] = "pandas",
|
|
27
|
+
date_dayfirst: bool = False,
|
|
28
|
+
reconciliation_tolerance: float = 0.05,
|
|
29
|
+
) -> Union[pd.DataFrame, CleaningReport]:
|
|
30
|
+
"""
|
|
31
|
+
Forensically parses messy Excel (.xlsx/.xls) or dirty CSV files.
|
|
32
|
+
|
|
33
|
+
Features:
|
|
34
|
+
- Strips arbitrary banner logos, disclaimers, and title rows.
|
|
35
|
+
- Handles multi-level & merged headers into clean snake_case identifiers.
|
|
36
|
+
- Intercepts and isolates embedded Subtotal and Grand Total rows.
|
|
37
|
+
- Coerces EU/US currencies, accounting parens '(100)', and mixed decimals.
|
|
38
|
+
- Performs mathematical reconciliation checks on dropped totals.
|
|
39
|
+
- Attaches an Audit Receipt.
|
|
40
|
+
"""
|
|
41
|
+
t0 = time.perf_counter()
|
|
42
|
+
raw_grid, detected_type = load_raw_grid_from_file(filepath_or_buffer, sheet_name=sheet_name)
|
|
43
|
+
|
|
44
|
+
if not raw_grid:
|
|
45
|
+
empty_df = pd.DataFrame()
|
|
46
|
+
if return_report:
|
|
47
|
+
rc = AuditReceipt(
|
|
48
|
+
source=str(filepath_or_buffer),
|
|
49
|
+
sheet_name=str(sheet_name) if detected_type == "xlsx" else None,
|
|
50
|
+
raw_shape=(0, 0),
|
|
51
|
+
clean_shape=(0, 0),
|
|
52
|
+
header_rows_detected=[],
|
|
53
|
+
execution_time_ms=0.0,
|
|
54
|
+
)
|
|
55
|
+
return CleaningReport(data=empty_df, receipt=rc)
|
|
56
|
+
return empty_df
|
|
57
|
+
|
|
58
|
+
raw_rows = len(raw_grid)
|
|
59
|
+
raw_cols = max(len(r) for r in raw_grid)
|
|
60
|
+
|
|
61
|
+
# 1. Detect headers and normalize columns
|
|
62
|
+
header_rows, col_names = HeaderDetector.detect_header_block(raw_grid)
|
|
63
|
+
header_end_idx = max(header_rows) if header_rows else 0
|
|
64
|
+
|
|
65
|
+
dropped_rows: List[DroppedRowInfo] = []
|
|
66
|
+
clean_records: List[Dict[str, Any]] = []
|
|
67
|
+
numeric_records_for_recon: List[Dict[str, float]] = []
|
|
68
|
+
|
|
69
|
+
# 2. Iterate through all rows and classify
|
|
70
|
+
for r_idx, row in enumerate(raw_grid):
|
|
71
|
+
if r_idx in header_rows:
|
|
72
|
+
continue
|
|
73
|
+
category = RowClassifier.classify_row(
|
|
74
|
+
row=row,
|
|
75
|
+
row_idx=r_idx,
|
|
76
|
+
header_end_idx=header_end_idx,
|
|
77
|
+
total_rows=raw_rows,
|
|
78
|
+
col_names=col_names,
|
|
79
|
+
)
|
|
80
|
+
|
|
81
|
+
row_preview = " | ".join([str(c) for c in row if c is not None and str(c).strip() != ""][:6])
|
|
82
|
+
|
|
83
|
+
if category in ("TITLE_BANNER", "BLANK_PADDING", "FOOTNOTE"):
|
|
84
|
+
dropped_rows.append(
|
|
85
|
+
DroppedRowInfo(row_index=r_idx, category=category, preview=row_preview)
|
|
86
|
+
)
|
|
87
|
+
continue
|
|
88
|
+
|
|
89
|
+
if category in ("SUBTOTAL", "GRAND_TOTAL"):
|
|
90
|
+
numerics = RowClassifier.extract_row_numerics(row, col_names)
|
|
91
|
+
dropped_rows.append(
|
|
92
|
+
DroppedRowInfo(
|
|
93
|
+
row_index=r_idx,
|
|
94
|
+
category=category,
|
|
95
|
+
preview=row_preview,
|
|
96
|
+
numeric_values=numerics,
|
|
97
|
+
)
|
|
98
|
+
)
|
|
99
|
+
continue
|
|
100
|
+
|
|
101
|
+
# Valid DATA Row: Perform forensic type normalization
|
|
102
|
+
record: Dict[str, Any] = {}
|
|
103
|
+
recon_numerics: Dict[str, float] = {}
|
|
104
|
+
|
|
105
|
+
for c_idx, col_name in enumerate(col_names):
|
|
106
|
+
val = row[c_idx] if c_idx < len(row) else None
|
|
107
|
+
# Check number first
|
|
108
|
+
parsed_num, is_num = parse_forensic_number(val)
|
|
109
|
+
if is_num:
|
|
110
|
+
record[col_name] = parsed_num
|
|
111
|
+
if parsed_num is not None:
|
|
112
|
+
recon_numerics[col_name] = float(parsed_num)
|
|
113
|
+
else:
|
|
114
|
+
# Check date
|
|
115
|
+
parsed_dt, is_dt = parse_forensic_date(val, dayfirst=date_dayfirst)
|
|
116
|
+
if is_dt:
|
|
117
|
+
record[col_name] = parsed_dt
|
|
118
|
+
else:
|
|
119
|
+
record[col_name] = val.strip() if isinstance(val, str) else val
|
|
120
|
+
|
|
121
|
+
clean_records.append(record)
|
|
122
|
+
numeric_records_for_recon.append(recon_numerics)
|
|
123
|
+
|
|
124
|
+
# 3. Create DataFrame
|
|
125
|
+
df = pd.DataFrame(clean_records)
|
|
126
|
+
|
|
127
|
+
# 4. Mathematical Invariant Verification
|
|
128
|
+
reconciliations = MathematicalReconciler.verify(
|
|
129
|
+
dropped_totals=dropped_rows,
|
|
130
|
+
clean_data_rows=numeric_records_for_recon,
|
|
131
|
+
tolerance=reconciliation_tolerance,
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
# 5. Build column transformation metadata
|
|
135
|
+
col_transforms: List[ColumnTransformation] = []
|
|
136
|
+
for c_idx, c_name in enumerate(col_names):
|
|
137
|
+
orig_val = raw_grid[header_rows[0]][c_idx] if header_rows and c_idx < len(raw_grid[header_rows[0]]) else ""
|
|
138
|
+
sample_before = [str(raw_grid[r][c_idx]) for r in range(header_end_idx + 1, min(header_end_idx + 4, raw_rows)) if c_idx < len(raw_grid[r])]
|
|
139
|
+
sample_after = df[c_name].dropna().head(3).tolist() if c_name in df.columns else []
|
|
140
|
+
col_transforms.append(
|
|
141
|
+
ColumnTransformation(
|
|
142
|
+
original_name=str(orig_val),
|
|
143
|
+
cleaned_name=c_name,
|
|
144
|
+
inferred_type=str(df[c_name].dtype) if c_name in df.columns else "unknown",
|
|
145
|
+
sample_values_before=sample_before,
|
|
146
|
+
sample_values_after=sample_after,
|
|
147
|
+
)
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
t1 = time.perf_counter()
|
|
151
|
+
receipt = AuditReceipt(
|
|
152
|
+
source=str(filepath_or_buffer),
|
|
153
|
+
sheet_name=str(sheet_name) if detected_type == "xlsx" else None,
|
|
154
|
+
raw_shape=(raw_rows, raw_cols),
|
|
155
|
+
clean_shape=df.shape,
|
|
156
|
+
header_rows_detected=header_rows,
|
|
157
|
+
dropped_rows=dropped_rows,
|
|
158
|
+
reconciliations=reconciliations,
|
|
159
|
+
column_transforms=col_transforms,
|
|
160
|
+
execution_time_ms=(t1 - t0) * 1000.0,
|
|
161
|
+
)
|
|
162
|
+
|
|
163
|
+
# Convert to Polars if requested
|
|
164
|
+
final_data: Any = df
|
|
165
|
+
if engine == "polars":
|
|
166
|
+
try:
|
|
167
|
+
import polars as pl
|
|
168
|
+
final_data = pl.from_pandas(df)
|
|
169
|
+
except ImportError:
|
|
170
|
+
raise ImportError("polars is not installed. Run `pip install polars` or use engine='pandas'.")
|
|
171
|
+
|
|
172
|
+
if return_report:
|
|
173
|
+
return CleaningReport(data=final_data, receipt=receipt)
|
|
174
|
+
|
|
175
|
+
return final_data
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def unmess(filepath_or_buffer: Any, **kwargs: Any) -> pd.DataFrame:
|
|
179
|
+
"""Convenience alias for read_messy."""
|
|
180
|
+
return read_messy(filepath_or_buffer, return_report=False, **kwargs)
|
sheet_unmess/detector.py
ADDED
|
@@ -0,0 +1,164 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Header Boundary and Merged Header Unstacker.
|
|
3
|
+
Detects title banners, unmerges openpyxl blocks, and combines multi-line headers.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
import io
|
|
8
|
+
import csv
|
|
9
|
+
from typing import Any, List, Tuple
|
|
10
|
+
import openpyxl
|
|
11
|
+
from sheet_unmess.normalizer import sanitize_column_name, parse_forensic_number
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def load_raw_grid_from_file(file_path_or_buffer: Any, sheet_name: Any = 0) -> Tuple[List[List[Any]], str]:
|
|
15
|
+
"""
|
|
16
|
+
Loads raw 2D grid preserving merged cell values across cells.
|
|
17
|
+
Returns (grid, file_type).
|
|
18
|
+
"""
|
|
19
|
+
if hasattr(file_path_or_buffer, "read"):
|
|
20
|
+
content = file_path_or_buffer.read()
|
|
21
|
+
is_bytes = isinstance(content, bytes)
|
|
22
|
+
stream = io.BytesIO(content) if is_bytes else io.StringIO(content)
|
|
23
|
+
# Attempt Excel load
|
|
24
|
+
try:
|
|
25
|
+
return _load_excel(stream, sheet_name), "xlsx"
|
|
26
|
+
except Exception:
|
|
27
|
+
if is_bytes:
|
|
28
|
+
stream = io.StringIO(content.decode("utf-8", errors="replace"))
|
|
29
|
+
else:
|
|
30
|
+
stream.seek(0)
|
|
31
|
+
return _load_csv(stream), "csv"
|
|
32
|
+
|
|
33
|
+
# String path
|
|
34
|
+
path_str = str(file_path_or_buffer).lower()
|
|
35
|
+
if path_str.endswith((".xlsx", ".xlsm", ".xltx", ".xltm")):
|
|
36
|
+
with open(file_path_or_buffer, "rb") as f:
|
|
37
|
+
return _load_excel(f, sheet_name), "xlsx"
|
|
38
|
+
else:
|
|
39
|
+
# Default CSV/TSV
|
|
40
|
+
with open(file_path_or_buffer, "r", encoding="utf-8", errors="replace") as f:
|
|
41
|
+
return _load_csv(f), "csv"
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _load_excel(stream_or_path: Any, sheet_name: Any = 0) -> List[List[Any]]:
|
|
45
|
+
wb = openpyxl.load_workbook(stream_or_path, data_only=True)
|
|
46
|
+
if isinstance(sheet_name, int):
|
|
47
|
+
ws = wb.worksheets[sheet_name]
|
|
48
|
+
else:
|
|
49
|
+
ws = wb[sheet_name]
|
|
50
|
+
|
|
51
|
+
# Resolve merged cell ranges by propagating the top-left value
|
|
52
|
+
merged_ranges = list(ws.merged_cells.ranges)
|
|
53
|
+
for m_range in merged_ranges:
|
|
54
|
+
min_col, min_row, max_col, max_row = m_range.bounds
|
|
55
|
+
top_left_value = ws.cell(row=min_row, column=min_col).value
|
|
56
|
+
# Unmerge so openpyxl doesn't guard assignments
|
|
57
|
+
ws.unmerge_cells(str(m_range))
|
|
58
|
+
for r in range(min_row, max_row + 1):
|
|
59
|
+
for c in range(min_col, max_col + 1):
|
|
60
|
+
ws.cell(row=r, column=c, value=top_left_value)
|
|
61
|
+
|
|
62
|
+
grid: List[List[Any]] = []
|
|
63
|
+
for row in ws.iter_rows(values_only=True):
|
|
64
|
+
grid.append(list(row))
|
|
65
|
+
return grid
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _load_csv(stream: Any) -> List[List[Any]]:
|
|
69
|
+
sample = stream.read(8192)
|
|
70
|
+
stream.seek(0)
|
|
71
|
+
try:
|
|
72
|
+
dialect = csv.Sniffer().sniff(sample)
|
|
73
|
+
delimiter = dialect.delimiter
|
|
74
|
+
except Exception:
|
|
75
|
+
delimiter = ","
|
|
76
|
+
|
|
77
|
+
reader = csv.reader(stream, delimiter=delimiter)
|
|
78
|
+
grid = [list(row) for row in reader]
|
|
79
|
+
return grid
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class HeaderDetector:
|
|
83
|
+
@staticmethod
|
|
84
|
+
def detect_header_block(grid: List[List[Any]]) -> Tuple[List[int], List[str]]:
|
|
85
|
+
"""
|
|
86
|
+
Calculates string vs numeric density and identifies header boundaries.
|
|
87
|
+
Supports multi-level stacked headers (e.g. Row 2: 2024, Row 3: Revenue).
|
|
88
|
+
"""
|
|
89
|
+
if not grid:
|
|
90
|
+
return [], []
|
|
91
|
+
|
|
92
|
+
max_cols = max(len(r) for r in grid)
|
|
93
|
+
# Normalize row lengths
|
|
94
|
+
normalized_grid = [r + [None] * (max_cols - len(r)) for r in grid]
|
|
95
|
+
|
|
96
|
+
# Score the first 20 rows
|
|
97
|
+
best_header_row = 0
|
|
98
|
+
max_string_density = -1.0
|
|
99
|
+
|
|
100
|
+
for r_idx in range(min(20, len(normalized_grid))):
|
|
101
|
+
row = normalized_grid[r_idx]
|
|
102
|
+
non_empty = [c for c in row if c is not None and str(c).strip() != ""]
|
|
103
|
+
if not non_empty:
|
|
104
|
+
continue
|
|
105
|
+
# A merged title banner is propagated across its merged range by
|
|
106
|
+
# the Excel loader; identical repeated values are not a real header.
|
|
107
|
+
if len(non_empty) > 1 and len({str(c).strip() for c in non_empty}) == 1:
|
|
108
|
+
continue
|
|
109
|
+
|
|
110
|
+
num_count = sum(1 for c in non_empty if parse_forensic_number(c)[1])
|
|
111
|
+
str_count = len(non_empty) - num_count
|
|
112
|
+
density = str_count / max(len(non_empty), 1)
|
|
113
|
+
|
|
114
|
+
# A header row is dominated by strings and spans a broad width
|
|
115
|
+
breadth_factor = len(non_empty) / max_cols
|
|
116
|
+
score = (density * 0.7) + (breadth_factor * 0.3)
|
|
117
|
+
|
|
118
|
+
if score > max_string_density and str_count > 0:
|
|
119
|
+
max_string_density = score
|
|
120
|
+
best_header_row = r_idx
|
|
121
|
+
|
|
122
|
+
# Check if the row immediately following is a sub-header (stacked)
|
|
123
|
+
header_rows = [best_header_row]
|
|
124
|
+
# Include an immediately preceding grouped header row (e.g. repeated
|
|
125
|
+
# year/quarter labels) when it contains repeated labels across columns.
|
|
126
|
+
if best_header_row > 0:
|
|
127
|
+
previous = [c for c in normalized_grid[best_header_row - 1] if c is not None and str(c).strip()]
|
|
128
|
+
previous_labels = [str(c).strip() for c in previous]
|
|
129
|
+
if len(previous_labels) > 1 and 1 < len(set(previous_labels)) < len(previous_labels):
|
|
130
|
+
header_rows.insert(0, best_header_row - 1)
|
|
131
|
+
next_row_idx = best_header_row + 1
|
|
132
|
+
if next_row_idx < len(normalized_grid):
|
|
133
|
+
next_row = normalized_grid[next_row_idx]
|
|
134
|
+
non_empty_next = [c for c in next_row if c is not None and str(c).strip() != ""]
|
|
135
|
+
if non_empty_next:
|
|
136
|
+
num_count = sum(1 for c in non_empty_next if parse_forensic_number(c)[1])
|
|
137
|
+
# If the next row is also predominantly strings, it's a stacked header
|
|
138
|
+
if num_count / len(non_empty_next) < 0.25 and len(non_empty_next) > 1:
|
|
139
|
+
header_rows.append(next_row_idx)
|
|
140
|
+
|
|
141
|
+
# Unstack and synthesize column names
|
|
142
|
+
final_cols: List[str] = []
|
|
143
|
+
for c in range(max_cols):
|
|
144
|
+
tokens = []
|
|
145
|
+
for h_idx in header_rows:
|
|
146
|
+
val = normalized_grid[h_idx][c]
|
|
147
|
+
if val is not None and str(val).strip():
|
|
148
|
+
tokens.append(str(val).strip())
|
|
149
|
+
|
|
150
|
+
raw_combo = "_".join(tokens) if tokens else f"unnamed_{c}"
|
|
151
|
+
final_cols.append(sanitize_column_name(raw_combo, c))
|
|
152
|
+
|
|
153
|
+
# Deduplicate names
|
|
154
|
+
seen = {}
|
|
155
|
+
unique_cols = []
|
|
156
|
+
for name in final_cols:
|
|
157
|
+
if name not in seen:
|
|
158
|
+
seen[name] = 0
|
|
159
|
+
unique_cols.append(name)
|
|
160
|
+
else:
|
|
161
|
+
seen[name] += 1
|
|
162
|
+
unique_cols.append(f"{name}_{seen[name]}")
|
|
163
|
+
|
|
164
|
+
return header_rows, unique_cols
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Native Model Context Protocol (MCP) Server for sheet-unmess.
|
|
3
|
+
Provides direct LLM tools for Claude Desktop, Cursor, Windsurf, and Cline.
|
|
4
|
+
Standard JSON-RPC 2.0 Stdio Implementation (Zero External Server Dependencies).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
import sys
|
|
9
|
+
import json
|
|
10
|
+
import traceback
|
|
11
|
+
from typing import Any, Dict
|
|
12
|
+
from sheet_unmess.core import read_messy
|
|
13
|
+
from sheet_unmess.receipt import CleaningReport
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
TOOLS = [
|
|
17
|
+
{
|
|
18
|
+
"name": "clean_spreadsheet",
|
|
19
|
+
"description": (
|
|
20
|
+
"Ingests and cleans messy corporate Excel (.xlsx) or CSV files. "
|
|
21
|
+
"Removes title banners, merged header noise, subtotal/grand total double-counting, "
|
|
22
|
+
"and normalizes EU/US numbers, returning clean tabular data and a mathematical audit receipt."
|
|
23
|
+
),
|
|
24
|
+
"inputSchema": {
|
|
25
|
+
"type": "object",
|
|
26
|
+
"properties": {
|
|
27
|
+
"file_path": {"type": "string", "description": "Absolute path to the spreadsheet."},
|
|
28
|
+
"sheet_name": {"type": ["string", "integer"], "default": 0, "description": "Sheet name or index."},
|
|
29
|
+
"date_dayfirst": {"type": "boolean", "default": False, "description": "Parse dates day-first (EU format)."},
|
|
30
|
+
"limit_rows": {"type": "integer", "default": 100, "description": "Number of preview rows to return in markdown format."}
|
|
31
|
+
},
|
|
32
|
+
"required": ["file_path"]
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
"name": "get_audit_receipt",
|
|
37
|
+
"description": "Returns the tamper-evident mathematical audit receipt for a cleaned spreadsheet.",
|
|
38
|
+
"inputSchema": {
|
|
39
|
+
"type": "object",
|
|
40
|
+
"properties": {
|
|
41
|
+
"file_path": {"type": "string", "description": "Absolute path to spreadsheet."},
|
|
42
|
+
"sheet_name": {"type": ["string", "integer"], "default": 0}
|
|
43
|
+
},
|
|
44
|
+
"required": ["file_path"]
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def handle_tool_call(name: str, arguments: Dict[str, Any]) -> Dict[str, Any]:
|
|
51
|
+
file_path = arguments.get("file_path")
|
|
52
|
+
sheet_name = arguments.get("sheet_name", 0)
|
|
53
|
+
dayfirst = arguments.get("date_dayfirst", False)
|
|
54
|
+
|
|
55
|
+
try:
|
|
56
|
+
report: CleaningReport = read_messy(
|
|
57
|
+
file_path,
|
|
58
|
+
sheet_name=sheet_name,
|
|
59
|
+
return_report=True,
|
|
60
|
+
date_dayfirst=dayfirst
|
|
61
|
+
)
|
|
62
|
+
except Exception as e:
|
|
63
|
+
return {
|
|
64
|
+
"content": [
|
|
65
|
+
{
|
|
66
|
+
"type": "text",
|
|
67
|
+
"text": f"Error ingesting file {file_path}: {str(e)}\n{traceback.format_exc()}"
|
|
68
|
+
}
|
|
69
|
+
],
|
|
70
|
+
"isError": True
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
if name == "get_audit_receipt":
|
|
74
|
+
return {
|
|
75
|
+
"content": [
|
|
76
|
+
{"type": "text", "text": report.receipt.to_markdown()}
|
|
77
|
+
]
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
elif name == "clean_spreadsheet":
|
|
81
|
+
limit = arguments.get("limit_rows", 100)
|
|
82
|
+
df = report.data.head(limit)
|
|
83
|
+
md_table = df.to_markdown(index=False)
|
|
84
|
+
receipt_md = report.receipt.to_markdown()
|
|
85
|
+
|
|
86
|
+
summary_text = (
|
|
87
|
+
f"### Cleaned Spreadsheet Preview (First {limit} rows)\n\n"
|
|
88
|
+
f"{md_table}\n\n"
|
|
89
|
+
f"---\n\n"
|
|
90
|
+
f"### Forensic Audit Summary\n"
|
|
91
|
+
f"{receipt_md}"
|
|
92
|
+
)
|
|
93
|
+
return {
|
|
94
|
+
"content": [{"type": "text", "text": summary_text}]
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
else:
|
|
98
|
+
raise ValueError(f"Unknown tool: {name}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def main():
|
|
102
|
+
"""Stdio JSON-RPC loop for Model Context Protocol."""
|
|
103
|
+
for line in sys.stdin:
|
|
104
|
+
if not line.strip():
|
|
105
|
+
continue
|
|
106
|
+
try:
|
|
107
|
+
req = json.loads(line)
|
|
108
|
+
except Exception:
|
|
109
|
+
continue
|
|
110
|
+
|
|
111
|
+
req_id = req.get("id")
|
|
112
|
+
method = req.get("method")
|
|
113
|
+
params = req.get("params", {})
|
|
114
|
+
|
|
115
|
+
if method == "initialize":
|
|
116
|
+
res = {
|
|
117
|
+
"jsonrpc": "2.0",
|
|
118
|
+
"id": req_id,
|
|
119
|
+
"result": {
|
|
120
|
+
"protocolVersion": "2024-11-05",
|
|
121
|
+
"capabilities": {"tools": {}},
|
|
122
|
+
"serverInfo": {"name": "sheet-unmess-mcp", "version": "1.0.0"}
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
elif method == "notifications/initialized":
|
|
126
|
+
continue
|
|
127
|
+
elif method == "tools/list":
|
|
128
|
+
res = {
|
|
129
|
+
"jsonrpc": "2.0",
|
|
130
|
+
"id": req_id,
|
|
131
|
+
"result": {"tools": TOOLS}
|
|
132
|
+
}
|
|
133
|
+
elif method == "tools/call":
|
|
134
|
+
tool_name = params.get("name")
|
|
135
|
+
tool_args = params.get("arguments", {})
|
|
136
|
+
call_res = handle_tool_call(tool_name, tool_args)
|
|
137
|
+
res = {
|
|
138
|
+
"jsonrpc": "2.0",
|
|
139
|
+
"id": req_id,
|
|
140
|
+
"result": call_res
|
|
141
|
+
}
|
|
142
|
+
else:
|
|
143
|
+
res = {
|
|
144
|
+
"jsonrpc": "2.0",
|
|
145
|
+
"id": req_id,
|
|
146
|
+
"error": {"code": -32601, "message": f"Method {method} not found"}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
sys.stdout.write(json.dumps(res) + "\n")
|
|
150
|
+
sys.stdout.flush()
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
if __name__ == "__main__":
|
|
154
|
+
main()
|
|
@@ -0,0 +1,125 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Forensic parser for EU/US mixed currency, decimals, accounting parentheses,
|
|
3
|
+
and mixed date formatting.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
import re
|
|
8
|
+
from typing import Any, Tuple
|
|
9
|
+
from dateutil import parser as date_parser
|
|
10
|
+
|
|
11
|
+
# Regex patterns
|
|
12
|
+
RE_ACCOUNTING_PARENS = re.compile(r"^\s*\(\s*([0-9.,\s]+)\s*\)\s*[$€£¥CHFchf]*\s*$")
|
|
13
|
+
RE_CURRENCY_SYMBOLS = re.compile(r"[$€£¥₹\s]|CHF|EUR|USD|GBP", re.IGNORECASE)
|
|
14
|
+
RE_CLEAN_NUM = re.compile(r"[^0-9,.-]")
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def parse_forensic_number(val: Any) -> Tuple[Any, bool]:
|
|
18
|
+
"""
|
|
19
|
+
Parses a string into a float/int handling:
|
|
20
|
+
- Accounting format: (1,234.50) -> -1234.50
|
|
21
|
+
- EU decimal commas: 1.234,56 -> 1234.56
|
|
22
|
+
- Mixed spacing: 1 234,50 -> 1234.50
|
|
23
|
+
- Percentage: 14.5% -> 0.145
|
|
24
|
+
Returns (parsed_value, is_numeric).
|
|
25
|
+
"""
|
|
26
|
+
if val is None:
|
|
27
|
+
return None, False
|
|
28
|
+
if isinstance(val, (int, float)):
|
|
29
|
+
return val, True
|
|
30
|
+
|
|
31
|
+
s = str(val).strip()
|
|
32
|
+
if not s or s.lower() in ("nan", "none", "null", "n/a", "#n/a", "-", "—"):
|
|
33
|
+
return None, False
|
|
34
|
+
|
|
35
|
+
is_negative = False
|
|
36
|
+
|
|
37
|
+
# Check for accounting parentheses e.g. (1,500.00)
|
|
38
|
+
paren_match = RE_ACCOUNTING_PARENS.match(s)
|
|
39
|
+
if paren_match:
|
|
40
|
+
is_negative = True
|
|
41
|
+
s = paren_match.group(1).strip()
|
|
42
|
+
elif s.startswith("-"):
|
|
43
|
+
is_negative = True
|
|
44
|
+
s = s[1:].strip()
|
|
45
|
+
|
|
46
|
+
is_percent = False
|
|
47
|
+
if s.endswith("%"):
|
|
48
|
+
is_percent = True
|
|
49
|
+
s = s[:-1].strip()
|
|
50
|
+
|
|
51
|
+
# Remove currency codes & symbols
|
|
52
|
+
s = RE_CURRENCY_SYMBOLS.sub("", s).strip()
|
|
53
|
+
# Strip any extra unexpected characters
|
|
54
|
+
s = RE_CLEAN_NUM.sub("", s).strip()
|
|
55
|
+
|
|
56
|
+
if not s:
|
|
57
|
+
return None, False
|
|
58
|
+
|
|
59
|
+
# Discern between US (1,234.56) and EU (1.234,56)
|
|
60
|
+
dot_count = s.count(".")
|
|
61
|
+
comma_count = s.count(",")
|
|
62
|
+
|
|
63
|
+
try:
|
|
64
|
+
if dot_count > 0 and comma_count > 0:
|
|
65
|
+
# Both present: the last one is the decimal separator
|
|
66
|
+
last_dot = s.rfind(".")
|
|
67
|
+
last_comma = s.rfind(",")
|
|
68
|
+
if last_comma > last_dot:
|
|
69
|
+
# EU format: 1.234.567,89
|
|
70
|
+
s = s.replace(".", "").replace(",", ".")
|
|
71
|
+
else:
|
|
72
|
+
# US format: 1,234,567.89
|
|
73
|
+
s = s.replace(",", "")
|
|
74
|
+
elif comma_count > 0 and dot_count == 0:
|
|
75
|
+
# Only commas. Check if it's EU decimal (e.g. 123,45) or US thousands (1,234)
|
|
76
|
+
parts = s.split(",")
|
|
77
|
+
if len(parts) == 2 and len(parts[1]) in (1, 2):
|
|
78
|
+
# High probability EU decimal: 45,5 or 1234,50
|
|
79
|
+
s = s.replace(",", ".")
|
|
80
|
+
elif all(len(p) == 3 for p in parts[1:]):
|
|
81
|
+
# US thousands separator: 1,000 or 1,234,567
|
|
82
|
+
s = s.replace(",", "")
|
|
83
|
+
else:
|
|
84
|
+
# Fallback to standard decimal
|
|
85
|
+
s = s.replace(",", ".")
|
|
86
|
+
elif dot_count > 1:
|
|
87
|
+
# Thousands separator with dots: 1.234.567
|
|
88
|
+
s = s.replace(".", "")
|
|
89
|
+
|
|
90
|
+
num = float(s)
|
|
91
|
+
if is_negative:
|
|
92
|
+
num = -num
|
|
93
|
+
if is_percent:
|
|
94
|
+
num = num / 100.0
|
|
95
|
+
return num, True
|
|
96
|
+
except ValueError:
|
|
97
|
+
return val, False
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def parse_forensic_date(val: Any, dayfirst: bool = False) -> Tuple[Any, bool]:
|
|
101
|
+
"""Tries parsing date while suppressing invalid numeric cascades."""
|
|
102
|
+
if val is None:
|
|
103
|
+
return None, False
|
|
104
|
+
s = str(val).strip()
|
|
105
|
+
if len(s) < 6:
|
|
106
|
+
return val, False
|
|
107
|
+
|
|
108
|
+
try:
|
|
109
|
+
dt = date_parser.parse(s, dayfirst=dayfirst, fuzzy=False)
|
|
110
|
+
return dt, True
|
|
111
|
+
except Exception:
|
|
112
|
+
return val, False
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def sanitize_column_name(raw: str, index: int) -> str:
|
|
116
|
+
"""Transforms arbitrary stacked header strings into clean Pythonic snake_case names."""
|
|
117
|
+
if not raw or not str(raw).strip():
|
|
118
|
+
return f"col_{index}"
|
|
119
|
+
s = str(raw).strip()
|
|
120
|
+
# Replace linebreaks and special chars with underscores
|
|
121
|
+
s = re.sub(r"[\r\n\t]+", "_", s)
|
|
122
|
+
s = re.sub(r"[^\w\s_]", "", s)
|
|
123
|
+
s = re.sub(r"[\s_]+", "_", s)
|
|
124
|
+
s = s.strip("_").lower()
|
|
125
|
+
return s if s else f"col_{index}"
|
sheet_unmess/receipt.py
ADDED
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Audit Receipt Data Structures and Visual Markdown / JSON Generators.
|
|
3
|
+
Guarantees mathematical transparency for forensic compliance.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
import json
|
|
8
|
+
from dataclasses import dataclass, field, asdict
|
|
9
|
+
from typing import Any, Dict, List, Optional
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
@dataclass
|
|
13
|
+
class ReconciledMetric:
|
|
14
|
+
column: str
|
|
15
|
+
reported_total: float
|
|
16
|
+
computed_sum: float
|
|
17
|
+
difference: float
|
|
18
|
+
row_index: int
|
|
19
|
+
status: str # "VERIFIED_EXACT", "ROUNDING_TOLERANCE", "DISCREPANCY"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class DroppedRowInfo:
|
|
24
|
+
row_index: int
|
|
25
|
+
category: str # "TITLE_BANNER", "SUBTOTAL", "GRAND_TOTAL", "FOOTNOTE", "BLANK_PADDING"
|
|
26
|
+
preview: str
|
|
27
|
+
numeric_values: Dict[str, float] = field(default_factory=dict)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class ColumnTransformation:
|
|
32
|
+
original_name: str
|
|
33
|
+
cleaned_name: str
|
|
34
|
+
inferred_type: str
|
|
35
|
+
sample_values_before: List[str]
|
|
36
|
+
sample_values_after: List[Any]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@dataclass
|
|
40
|
+
class AuditReceipt:
|
|
41
|
+
source: str
|
|
42
|
+
sheet_name: Optional[str]
|
|
43
|
+
raw_shape: tuple[int, int]
|
|
44
|
+
clean_shape: tuple[int, int]
|
|
45
|
+
header_rows_detected: List[int]
|
|
46
|
+
dropped_rows: List[DroppedRowInfo] = field(default_factory=list)
|
|
47
|
+
reconciliations: List[ReconciledMetric] = field(default_factory=list)
|
|
48
|
+
column_transforms: List[ColumnTransformation] = field(default_factory=list)
|
|
49
|
+
execution_time_ms: float = 0.0
|
|
50
|
+
|
|
51
|
+
def to_dict(self) -> Dict[str, Any]:
|
|
52
|
+
return asdict(self)
|
|
53
|
+
|
|
54
|
+
def to_json(self, indent: int = 2) -> str:
|
|
55
|
+
return json.dumps(self.to_dict(), indent=indent, default=str)
|
|
56
|
+
|
|
57
|
+
def to_markdown(self) -> str:
|
|
58
|
+
md = []
|
|
59
|
+
md.append(f"# 🛡️ Forensic Audit Receipt: `{self.source}`")
|
|
60
|
+
if self.sheet_name:
|
|
61
|
+
md.append(f"**Sheet:** `{self.sheet_name}`")
|
|
62
|
+
md.append(f"- **Input Shape:** {self.raw_shape[0]} rows × {self.raw_shape[1]} cols")
|
|
63
|
+
md.append(f"- **Clean Output Shape:** {self.clean_shape[0]} rows × {self.clean_shape[1]} cols")
|
|
64
|
+
md.append(f"- **Execution Time:** {self.execution_time_ms:.2f} ms")
|
|
65
|
+
md.append(f"- **Header Rows Extracted:** {self.header_rows_detected}")
|
|
66
|
+
md.append("")
|
|
67
|
+
|
|
68
|
+
# Reconciliations
|
|
69
|
+
md.append("## 1. Mathematical Invariant Reconciliation")
|
|
70
|
+
if not self.reconciliations:
|
|
71
|
+
md.append("*(No embedded subtotal or grand total rows found for mathematical verification)*")
|
|
72
|
+
else:
|
|
73
|
+
md.append("| Column | Dropped Total | Recomputed Line Item Sum | Difference | Audit Status |")
|
|
74
|
+
md.append("|---|---|---|---|---|")
|
|
75
|
+
for r in self.reconciliations:
|
|
76
|
+
icon = "✅" if r.status in ("VERIFIED_EXACT", "ROUNDING_TOLERANCE") else "🚨"
|
|
77
|
+
md.append(
|
|
78
|
+
f"| **{r.column}** | {r.reported_total:,.2f} | {r.computed_sum:,.2f} | {r.difference:,.4f} | {icon} `{r.status}` |"
|
|
79
|
+
)
|
|
80
|
+
md.append("")
|
|
81
|
+
|
|
82
|
+
# Dropped Rows Breakdown
|
|
83
|
+
md.append("## 2. Forensic Row Interceptions")
|
|
84
|
+
cats: Dict[str, int] = {}
|
|
85
|
+
for dr in self.dropped_rows:
|
|
86
|
+
cats[dr.category] = cats.get(dr.category, 0) + 1
|
|
87
|
+
|
|
88
|
+
md.append(f"Total Rows Intercepted: **{len(self.dropped_rows)}**")
|
|
89
|
+
for cat, count in cats.items():
|
|
90
|
+
md.append(f"- **{cat}**: {count} row(s)")
|
|
91
|
+
|
|
92
|
+
md.append("")
|
|
93
|
+
md.append("<details><summary>Click to expand dropped row details</summary>\n")
|
|
94
|
+
md.append("| Row # | Category | Preview |")
|
|
95
|
+
md.append("|---|---|---|")
|
|
96
|
+
for dr in self.dropped_rows[:50]: # limit 50 in markdown
|
|
97
|
+
preview = (dr.preview[:75] + "...") if len(dr.preview) > 75 else dr.preview
|
|
98
|
+
preview = preview.replace("|", "\\|").replace("\n", " ")
|
|
99
|
+
md.append(f"| {dr.row_index} | `{dr.category}` | `{preview}` |")
|
|
100
|
+
if len(self.dropped_rows) > 50:
|
|
101
|
+
md.append(f"| ... | ... | *(and {len(self.dropped_rows) - 50} more)* |")
|
|
102
|
+
md.append("</details>\n")
|
|
103
|
+
|
|
104
|
+
# Column Schema Transformations
|
|
105
|
+
md.append("## 3. Schema & Normalization Map")
|
|
106
|
+
md.append("| Clean Column | Original Header | Coerced Type | Sample Conversion |")
|
|
107
|
+
md.append("|---|---|---|---|")
|
|
108
|
+
for ct in self.column_transforms:
|
|
109
|
+
before = str(ct.sample_values_before[0]) if ct.sample_values_before else ""
|
|
110
|
+
after = str(ct.sample_values_after[0]) if ct.sample_values_after else ""
|
|
111
|
+
md.append(f"| `{ct.cleaned_name}` | `{ct.original_name}` | `{ct.inferred_type}` | `{before}` ➔ `{after}` |")
|
|
112
|
+
|
|
113
|
+
md.append("\n---\n*Generated by `sheet-unmess` - Forensic Ingestion Engine*")
|
|
114
|
+
return "\n".join(md)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
@dataclass
|
|
118
|
+
class CleaningReport:
|
|
119
|
+
"""Wrapper object returned when full report is requested."""
|
|
120
|
+
data: Any # pandas or polars DataFrame
|
|
121
|
+
receipt: AuditReceipt
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Forensic Mathematical Reconciliation Engine.
|
|
3
|
+
Verifies that removed subtotal/grand total rows match the sum of their constituent line items.
|
|
4
|
+
"""
|
|
5
|
+
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
from typing import Dict, List, Tuple
|
|
8
|
+
from sheet_unmess.receipt import ReconciledMetric, DroppedRowInfo
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class MathematicalReconciler:
|
|
12
|
+
@staticmethod
|
|
13
|
+
def verify(
|
|
14
|
+
dropped_totals: List[DroppedRowInfo],
|
|
15
|
+
clean_data_rows: List[Dict[str, float]],
|
|
16
|
+
tolerance: float = 0.05,
|
|
17
|
+
) -> List[ReconciledMetric]:
|
|
18
|
+
"""
|
|
19
|
+
Compares line-item sums against dropped totals.
|
|
20
|
+
Detects rounding drift vs critical balance discrepancies.
|
|
21
|
+
"""
|
|
22
|
+
reconciliations: List[ReconciledMetric] = []
|
|
23
|
+
if not dropped_totals or not clean_data_rows:
|
|
24
|
+
return reconciliations
|
|
25
|
+
|
|
26
|
+
# Aggregate sums of all clean line items
|
|
27
|
+
column_sums: Dict[str, float] = {}
|
|
28
|
+
for row in clean_data_rows:
|
|
29
|
+
for col, val in row.items():
|
|
30
|
+
column_sums[col] = column_sums.get(col, 0.0) + val
|
|
31
|
+
|
|
32
|
+
# Evaluate against grand totals first, then subtotals
|
|
33
|
+
grand_totals = [dt for dt in dropped_totals if dt.category == "GRAND_TOTAL"]
|
|
34
|
+
targets = grand_totals if grand_totals else [dt for dt in dropped_totals if dt.category == "SUBTOTAL"]
|
|
35
|
+
|
|
36
|
+
for target in targets:
|
|
37
|
+
for col, reported in target.numeric_values.items():
|
|
38
|
+
if col in column_sums:
|
|
39
|
+
computed = column_sums[col]
|
|
40
|
+
diff = abs(computed - reported)
|
|
41
|
+
|
|
42
|
+
if diff < 1e-4:
|
|
43
|
+
status = "VERIFIED_EXACT"
|
|
44
|
+
elif diff <= tolerance:
|
|
45
|
+
status = "ROUNDING_TOLERANCE"
|
|
46
|
+
else:
|
|
47
|
+
status = "DISCREPANCY"
|
|
48
|
+
|
|
49
|
+
reconciliations.append(
|
|
50
|
+
ReconciledMetric(
|
|
51
|
+
column=col,
|
|
52
|
+
reported_total=reported,
|
|
53
|
+
computed_sum=computed,
|
|
54
|
+
difference=diff,
|
|
55
|
+
row_index=target.row_index,
|
|
56
|
+
status=status,
|
|
57
|
+
)
|
|
58
|
+
)
|
|
59
|
+
return reconciliations
|
|
@@ -0,0 +1,139 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: sheet-unmess
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: Forensic Messy-Excel & Dirty-CSV Ingestor with Tamper-Evident Audit Receipts and Native MCP Server
|
|
5
|
+
Project-URL: Homepage, https://github.com/sheet-unmess/sheet-unmess
|
|
6
|
+
Project-URL: Documentation, https://github.com/sheet-unmess/sheet-unmess#readme
|
|
7
|
+
Project-URL: Repository, https://github.com/sheet-unmess/sheet-unmess
|
|
8
|
+
Author: Sheet Unmess Contributors
|
|
9
|
+
License: MIT
|
|
10
|
+
Keywords: audit,claude,csv,cursor,data-cleaning,excel,llm-agent,mcp,mcp-server,pandas,windsurf
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Intended Audience :: Financial and Insurance Industry
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Information Analysis
|
|
19
|
+
Requires-Python: >=3.10
|
|
20
|
+
Requires-Dist: openpyxl>=3.1.0
|
|
21
|
+
Requires-Dist: pandas>=2.0.0
|
|
22
|
+
Requires-Dist: python-dateutil>=2.8.2
|
|
23
|
+
Provides-Extra: all
|
|
24
|
+
Requires-Dist: polars>=0.20.0; extra == 'all'
|
|
25
|
+
Provides-Extra: polars
|
|
26
|
+
Requires-Dist: polars>=0.20.0; extra == 'polars'
|
|
27
|
+
Description-Content-Type: text/markdown
|
|
28
|
+
|
|
29
|
+
# sheet-unmess
|
|
30
|
+
|
|
31
|
+
Forensic-style ingestion for messy Excel (`.xlsx`) and CSV files. It attempts to
|
|
32
|
+
identify headers, remove common non-data rows, normalize common number formats,
|
|
33
|
+
and produce a human-readable audit receipt.
|
|
34
|
+
|
|
35
|
+
> **Important:** automatic header/row classification is heuristic. Always inspect
|
|
36
|
+
> the output and reconciliation receipt before using cleaned data for financial,
|
|
37
|
+
> regulatory, or other high-stakes decisions. A receipt is an audit report, not a
|
|
38
|
+
> cryptographic signature or proof that no data was lost.
|
|
39
|
+
|
|
40
|
+
## Features
|
|
41
|
+
|
|
42
|
+
- Detects likely header rows and combines adjacent stacked headers.
|
|
43
|
+
- Propagates values from the top-left cell across merged Excel ranges.
|
|
44
|
+
- Classifies common title, blank, footnote, subtotal, and grand-total rows.
|
|
45
|
+
- Parses common US/EU number formats, accounting negatives, and percentages.
|
|
46
|
+
- Optionally returns a cleaning report with dropped-row details and total checks.
|
|
47
|
+
- Provides a CLI and a stdio MCP server for compatible AI clients.
|
|
48
|
+
|
|
49
|
+
## Install from source
|
|
50
|
+
|
|
51
|
+
Python 3.10 or newer is required.
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
python -m venv .venv
|
|
55
|
+
# Windows PowerShell:
|
|
56
|
+
.venv\Scripts\Activate.ps1
|
|
57
|
+
# macOS/Linux:
|
|
58
|
+
source .venv/bin/activate
|
|
59
|
+
|
|
60
|
+
python -m pip install --upgrade pip
|
|
61
|
+
pip install -e .
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Quick start
|
|
65
|
+
|
|
66
|
+
```python
|
|
67
|
+
from sheet_unmess import read_messy
|
|
68
|
+
|
|
69
|
+
df = read_messy("quarterly_report.xlsx")
|
|
70
|
+
print(df.head())
|
|
71
|
+
|
|
72
|
+
report = read_messy("quarterly_report.xlsx", return_report=True)
|
|
73
|
+
print(report.data.head())
|
|
74
|
+
print(report.receipt.to_markdown())
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
For European day-first dates, pass `date_dayfirst=True`.
|
|
78
|
+
|
|
79
|
+
## CLI
|
|
80
|
+
|
|
81
|
+
```bash
|
|
82
|
+
sheet-unmess clean input.xlsx --out cleaned.csv --receipt audit.md
|
|
83
|
+
sheet-unmess clean input.csv --out cleaned.csv --dayfirst
|
|
84
|
+
sheet-unmess clean input.xlsx --sheet Financials --out cleaned.csv
|
|
85
|
+
sheet-unmess init-ai
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
`--out` writes CSV by default; use a `.parquet` extension for Parquet output
|
|
89
|
+
(and install a suitable Parquet engine such as `pyarrow`).
|
|
90
|
+
|
|
91
|
+
## Tests
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
python -m pip install -e ".[dev]"
|
|
95
|
+
python -m pytest
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
Or run the included test file directly:
|
|
99
|
+
|
|
100
|
+
```bash
|
|
101
|
+
python tests/test_sheet_unmess.py
|
|
102
|
+
```
|
|
103
|
+
|
|
104
|
+
## MCP server
|
|
105
|
+
|
|
106
|
+
The package exposes `sheet-unmess-mcp` and `sheet-unmess mcp`. Example generic
|
|
107
|
+
stdio MCP configuration:
|
|
108
|
+
|
|
109
|
+
```json
|
|
110
|
+
{
|
|
111
|
+
"mcpServers": {
|
|
112
|
+
"sheet-unmess": {
|
|
113
|
+
"command": "python",
|
|
114
|
+
"args": ["-m", "sheet_unmess.mcp_server"]
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
Ensure the `python` executable points to the environment where `sheet-unmess`
|
|
121
|
+
is installed. The `sheet-unmess init-ai` command writes project rules, a Cursor
|
|
122
|
+
MCP configuration, and attempts to update the Claude Desktop configuration.
|
|
123
|
+
Review generated configuration before using it.
|
|
124
|
+
|
|
125
|
+
## Current limitations
|
|
126
|
+
|
|
127
|
+
- `.xls` legacy Excel files are not supported by the current openpyxl loader.
|
|
128
|
+
- Header detection and row classification are heuristic and may need manual review.
|
|
129
|
+
- Reconciliation currently compares embedded totals against aggregated clean-row
|
|
130
|
+
sums; nested subtotals and multiple independent sections can require more
|
|
131
|
+
sophisticated grouping logic.
|
|
132
|
+
- Mixed ambiguous dates cannot be inferred reliably; use `date_dayfirst` when
|
|
133
|
+
appropriate.
|
|
134
|
+
- The MCP server reads files by local path and therefore needs filesystem access
|
|
135
|
+
to the file being processed.
|
|
136
|
+
|
|
137
|
+
## License
|
|
138
|
+
|
|
139
|
+
MIT
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
sheet_unmess/__init__.py,sha256=z79H2YxFtQ5Z9aYZZpm15fy3A7BZpMd1tnQ3uOgPKnU,379
|
|
2
|
+
sheet_unmess/ai_rules.py,sha256=7NhWoIssiBVI1pT_6n9919DKl6bjPVuv5PrSMSk4qEw,2713
|
|
3
|
+
sheet_unmess/classifier.py,sha256=nV79ovdwLviK3uloHzHWJtzfHAyYFr914JdmEzMT-LE,2934
|
|
4
|
+
sheet_unmess/cli.py,sha256=L3T3EyMC1efOmIMjPtRQqNPQqy7Ejm_dLuym5wCKXFE,3502
|
|
5
|
+
sheet_unmess/core.py,sha256=quJjT7c4Gp-4bPRR2p4ZC7dMYfzcP7U5IOlHc1BBc04,6654
|
|
6
|
+
sheet_unmess/detector.py,sha256=0M9YaWD5794OKmu5Pe17ddSNYuIdz7Mloe_YWubeGPw,6449
|
|
7
|
+
sheet_unmess/mcp_server.py,sha256=uRtBFB83ohD3iyFVE4SCobO9tX5huFKZmi3FUAvlOR0,4996
|
|
8
|
+
sheet_unmess/normalizer.py,sha256=HX3liIRMagXakKDV22hJf-d9cr4ECJLCvbnAFygeaLM,3998
|
|
9
|
+
sheet_unmess/receipt.py,sha256=qDcUFgj1UCzg_qSnbKom-MZAG6qwiFCIx8774qy1_Lk,4766
|
|
10
|
+
sheet_unmess/reconciler.py,sha256=zRxwDFqRPxdI96igZIflqgIip-IVaHc5FpjBPth3hjc,2245
|
|
11
|
+
sheet_unmess-1.0.0.dist-info/METADATA,sha256=nkBEkp9RuBNjQKbLV1rvLXr1aYmtp1uORPor4z2YVq0,4465
|
|
12
|
+
sheet_unmess-1.0.0.dist-info/WHEEL,sha256=2iZrJkDIkeDh0k9SikKMqFuRP-ua6mb4Ir1AsER4BEw,87
|
|
13
|
+
sheet_unmess-1.0.0.dist-info/entry_points.txt,sha256=23jNFmaq0BzBsKekS2CT49uMR7UuEGqZTPP1F-CG6WY,103
|
|
14
|
+
sheet_unmess-1.0.0.dist-info/RECORD,,
|