sophhub 0.4.19 → 0.4.21

package/skills/sophnet-customer-management/src/src/customer_mgmt_core/parser.py

@@ -1,241 +1,241 @@
-from __future__ import annotations
-
-import csv
-import re
-from pathlib import Path
-from typing import Any
-
-import openpyxl
-import xlrd
-
-from .normalizer import clean_text, is_blank
-from .schema import resolve_grouped_record_field
-
-
-def parse_file(file_path: str | Path) -> dict[str, Any]:
-    path = Path(file_path)
-    suffix = path.suffix.lower()
-    if suffix in {".csv"}:
-        tables = [_grid_to_table(_read_csv_grid(path), title=path.stem)]
-    elif suffix in {".xlsx"}:
-        tables = _read_xlsx_tables(path)
-    elif suffix in {".xls"}:
-        tables = _read_xls_tables(path)
-    elif suffix in {".md", ".markdown", ".txt"}:
-        tables = _read_markdown_tables(path)
-    else:
-        raise ValueError(f"暂不支持的文件类型: {suffix}")
-
-    filtered_tables = [table for table in tables if table and table["rows"]]
-    return {
-        "source_file": str(path),
-        "format": suffix.lstrip("."),
-        "tables": filtered_tables,
-    }
-
-
-def _read_csv_grid(path: Path) -> list[list[str]]:
-    with path.open("r", encoding="utf-8-sig", newline="") as file:
-        return [[clean_text(cell) for cell in row] for row in csv.reader(file)]
-
-
-def _read_xlsx_tables(path: Path) -> list[dict[str, Any]]:
-    workbook = openpyxl.load_workbook(path, data_only=True)
-    tables: list[dict[str, Any]] = []
-    for sheet in workbook.worksheets:
-        grid = [
-            [sheet.cell(row_idx, col_idx).value for col_idx in range(1, sheet.max_column + 1)]
-            for row_idx in range(1, sheet.max_row + 1)
-        ]
-        for merged in sheet.merged_cells.ranges:
-            value = grid[merged.min_row - 1][merged.min_col - 1]
-            for row_idx in range(merged.min_row - 1, merged.max_row):
-                for col_idx in range(merged.min_col - 1, merged.max_col):
-                    grid[row_idx][col_idx] = value
-        grid = [[clean_text(cell) for cell in row] for row in grid]
-        table = _grid_to_table(grid, title=sheet.title)
-        if table:
-            tables.append(table)
-    return tables
-
-
-def _read_xls_tables(path: Path) -> list[dict[str, Any]]:
-    workbook = xlrd.open_workbook(path, formatting_info=True)
-    tables: list[dict[str, Any]] = []
-    for index in range(workbook.nsheets):
-        sheet = workbook.sheet_by_index(index)
-        grid = [
-            [sheet.cell_value(row_idx, col_idx) for col_idx in range(sheet.ncols)]
-            for row_idx in range(sheet.nrows)
-        ]
-        for row_lo, row_hi, col_lo, col_hi in sheet.merged_cells:
-            value = grid[row_lo][col_lo]
-            for row_idx in range(row_lo, row_hi):
-                for col_idx in range(col_lo, col_hi):
-                    grid[row_idx][col_idx] = value
-        grid = [[clean_text(cell) for cell in row] for row in grid]
-        table = _grid_to_table(grid, title=sheet.name)
-        if table:
-            tables.append(table)
-    return tables
-
-
-def _read_markdown_tables(path: Path) -> list[dict[str, Any]]:
-    lines = path.read_text(encoding="utf-8").splitlines()
-    blocks: list[list[list[str]]] = []
-    current: list[list[str]] = []
-
-    def flush_current() -> None:
-        nonlocal current
-        if current:
-            blocks.append(current)
-            current = []
-
-    for line in lines:
-        stripped = line.strip()
-        if not stripped.startswith("|"):
-            flush_current()
-            continue
-        cells = [clean_text(cell) for cell in stripped.strip("|").split("|")]
-        if all(re.fullmatch(r"[-:\s]+", cell or "-") for cell in cells):
-            continue
-        current.append(cells)
-    flush_current()
-    tables = []
-    for index, block in enumerate(blocks):
-        table = _grid_to_table(block, title=f"table_{index + 1}")
-        if table:
-            tables.append(table)
-    return tables
-
-
-def _grid_to_table(grid: list[list[str]], title: str) -> dict[str, Any] | None:
-    rows = [_trim_row(row) for row in grid if any(not is_blank(cell) for cell in row)]
-    if not rows:
-        return None
-
-    title_row = None
-    if _looks_like_title_row(rows[0]):
-        title_row = rows.pop(0)
-    if not rows:
-        return None
-
-    header_start = 0
-    headers = rows[0]
-    data_start = 1
-    if _has_second_header_row(rows):
-        headers = _combine_headers(rows[0], rows[1])
-        data_start = 2
-    headers = _dedupe_headers(headers)
-    if not any(headers):
-        return None
-
-    mode = "grouped_records" if _looks_like_grouped_record_headers(headers) else "generic"
-    parsed_rows = _rows_to_dicts(rows[data_start:], headers, forward_fill=(mode == "grouped_records"))
-    return {
-        "title": clean_text(title_row[0]) if title_row else title,
-        "headers": headers,
-        "rows": parsed_rows,
-        "mode": mode,
-    }
-
-
-def _trim_row(row: list[str]) -> list[str]:
-    cleaned = [clean_text(cell) for cell in row]
-    while cleaned and is_blank(cleaned[-1]):
-        cleaned.pop()
-    return cleaned
-
-
-def _looks_like_title_row(row: list[str]) -> bool:
-    first = clean_text(row[0]) if row else ""
-    if "客户档案" in first:
-        return True
-    unnamed_count = sum(1 for cell in row if "unnamed" in clean_text(cell).lower())
-    return unnamed_count >= max(2, len(row) // 3)
-
-
-def _has_second_header_row(rows: list[list[str]]) -> bool:
-    if len(rows) < 2:
-        return False
-    second_row = rows[1]
-    markers = {"姓名", "性别", "生日"}
-    return sum(1 for cell in second_row if clean_text(cell) in markers) >= 2
-
-
-def _combine_headers(main_headers: list[str], sub_headers: list[str]) -> list[str]:
-    result: list[str] = []
-    current_group = ""
-    for index, main in enumerate(main_headers):
-        sub = sub_headers[index] if index < len(sub_headers) else ""
-        main_text = clean_text(main)
-        sub_text = clean_text(sub)
-        if not is_blank(main_text):
-            current_group = main_text
-        elif current_group and sub_text:
-            main_text = current_group
-        if is_blank(main_text):
-            result.append(sub_text)
-            continue
-        if is_blank(sub_text):
-            result.append(main_text)
-            continue
-        if sub_text == main_text:
-            result.append(main_text)
-            continue
-        result.append(f"{main_text}_{sub_text}")
-    return result
-
-
-def _dedupe_headers(headers: list[str]) -> list[str]:
-    seen: dict[str, int] = {}
-    result: list[str] = []
-    for index, header in enumerate(headers):
-        text = clean_text(header)
-        if not text or text.lower().startswith("unnamed"):
-            text = f"column_{index + 1}"
-        count = seen.get(text, 0)
-        seen[text] = count + 1
-        result.append(text if count == 0 else f"{text}_{count + 1}")
-    return result
-
-
-def _looks_like_grouped_record_headers(headers: list[str]) -> bool:
-    grouped_hits = 0
-    normalized_hits = 0
-    for header in headers:
-        if "_" in header:
-            prefix, suffix = header.split("_", 1)
-            if prefix and suffix in {"姓名", "性别", "生日", "出生日期", "手机", "手机号", "电话", "联系电话"}:
-                grouped_hits += 1
-        if resolve_grouped_record_field(header):
-            normalized_hits += 1
-    return grouped_hits >= 2 or normalized_hits >= 3
-
-
-def _rows_to_dicts(rows: list[list[str]], headers: list[str], *, forward_fill: bool) -> list[dict[str, str]]:
-    output: list[dict[str, str]] = []
-    previous_values = {header: "" for header in headers}
-    for raw_row in rows:
-        if not raw_row:
-            continue
-        row: dict[str, str] = {}
-        for index, header in enumerate(headers):
-            value = clean_text(raw_row[index]) if index < len(raw_row) else ""
-            if forward_fill and is_blank(value):
-                value = previous_values.get(header, "")
-            row[header] = value
-            if not is_blank(value):
-                previous_values[header] = value
-        if _is_meaningful_row(row):
-            output.append(row)
-    return output
-
-
-def _is_meaningful_row(row: dict[str, str]) -> bool:
-    non_blank_values = [value for value in row.values() if not is_blank(value)]
-    if not non_blank_values:
-        return False
-    if len(non_blank_values) == 1 and non_blank_values[0].isdigit():
-        return False
-    return True
+from __future__ import annotations
+
+import csv
+import re
+from pathlib import Path
+from typing import Any
+
+import openpyxl
+import xlrd
+
+from .normalizer import clean_text, is_blank
+from .schema import resolve_grouped_record_field
+
+
+def parse_file(file_path: str | Path) -> dict[str, Any]:
+    path = Path(file_path)
+    suffix = path.suffix.lower()
+    if suffix in {".csv"}:
+        tables = [_grid_to_table(_read_csv_grid(path), title=path.stem)]
+    elif suffix in {".xlsx"}:
+        tables = _read_xlsx_tables(path)
+    elif suffix in {".xls"}:
+        tables = _read_xls_tables(path)
+    elif suffix in {".md", ".markdown", ".txt"}:
+        tables = _read_markdown_tables(path)
+    else:
+        raise ValueError(f"暂不支持的文件类型: {suffix}")
+
+    filtered_tables = [table for table in tables if table and table["rows"]]
+    return {
+        "source_file": str(path),
+        "format": suffix.lstrip("."),
+        "tables": filtered_tables,
+    }
+
+
+def _read_csv_grid(path: Path) -> list[list[str]]:
+    with path.open("r", encoding="utf-8-sig", newline="") as file:
+        return [[clean_text(cell) for cell in row] for row in csv.reader(file)]
+
+
+def _read_xlsx_tables(path: Path) -> list[dict[str, Any]]:
+    workbook = openpyxl.load_workbook(path, data_only=True)
+    tables: list[dict[str, Any]] = []
+    for sheet in workbook.worksheets:
+        grid = [
+            [sheet.cell(row_idx, col_idx).value for col_idx in range(1, sheet.max_column + 1)]
+            for row_idx in range(1, sheet.max_row + 1)
+        ]
+        for merged in sheet.merged_cells.ranges:
+            value = grid[merged.min_row - 1][merged.min_col - 1]
+            for row_idx in range(merged.min_row - 1, merged.max_row):
+                for col_idx in range(merged.min_col - 1, merged.max_col):
+                    grid[row_idx][col_idx] = value
+        grid = [[clean_text(cell) for cell in row] for row in grid]
+        table = _grid_to_table(grid, title=sheet.title)
+        if table:
+            tables.append(table)
+    return tables
+
+
+def _read_xls_tables(path: Path) -> list[dict[str, Any]]:
+    workbook = xlrd.open_workbook(path, formatting_info=True)
+    tables: list[dict[str, Any]] = []
+    for index in range(workbook.nsheets):
+        sheet = workbook.sheet_by_index(index)
+        grid = [
+            [sheet.cell_value(row_idx, col_idx) for col_idx in range(sheet.ncols)]
+            for row_idx in range(sheet.nrows)
+        ]
+        for row_lo, row_hi, col_lo, col_hi in sheet.merged_cells:
+            value = grid[row_lo][col_lo]
+            for row_idx in range(row_lo, row_hi):
+                for col_idx in range(col_lo, col_hi):
+                    grid[row_idx][col_idx] = value
+        grid = [[clean_text(cell) for cell in row] for row in grid]
+        table = _grid_to_table(grid, title=sheet.name)
+        if table:
+            tables.append(table)
+    return tables
+
+
+def _read_markdown_tables(path: Path) -> list[dict[str, Any]]:
+    lines = path.read_text(encoding="utf-8").splitlines()
+    blocks: list[list[list[str]]] = []
+    current: list[list[str]] = []
+
+    def flush_current() -> None:
+        nonlocal current
+        if current:
+            blocks.append(current)
+            current = []
+
+    for line in lines:
+        stripped = line.strip()
+        if not stripped.startswith("|"):
+            flush_current()
+            continue
+        cells = [clean_text(cell) for cell in stripped.strip("|").split("|")]
+        if all(re.fullmatch(r"[-:\s]+", cell or "-") for cell in cells):
+            continue
+        current.append(cells)
+    flush_current()
+    tables = []
+    for index, block in enumerate(blocks):
+        table = _grid_to_table(block, title=f"table_{index + 1}")
+        if table:
+            tables.append(table)
+    return tables
+
+
+def _grid_to_table(grid: list[list[str]], title: str) -> dict[str, Any] | None:
+    rows = [_trim_row(row) for row in grid if any(not is_blank(cell) for cell in row)]
+    if not rows:
+        return None
+
+    title_row = None
+    if _looks_like_title_row(rows[0]):
+        title_row = rows.pop(0)
+    if not rows:
+        return None
+
+    header_start = 0
+    headers = rows[0]
+    data_start = 1
+    if _has_second_header_row(rows):
+        headers = _combine_headers(rows[0], rows[1])
+        data_start = 2
+    headers = _dedupe_headers(headers)
+    if not any(headers):
+        return None
+
+    mode = "grouped_records" if _looks_like_grouped_record_headers(headers) else "generic"
+    parsed_rows = _rows_to_dicts(rows[data_start:], headers, forward_fill=(mode == "grouped_records"))
+    return {
+        "title": clean_text(title_row[0]) if title_row else title,
+        "headers": headers,
+        "rows": parsed_rows,
+        "mode": mode,
+    }
+
+
+def _trim_row(row: list[str]) -> list[str]:
+    cleaned = [clean_text(cell) for cell in row]
+    while cleaned and is_blank(cleaned[-1]):
+        cleaned.pop()
+    return cleaned
+
+
+def _looks_like_title_row(row: list[str]) -> bool:
+    first = clean_text(row[0]) if row else ""
+    if "客户档案" in first:
+        return True
+    unnamed_count = sum(1 for cell in row if "unnamed" in clean_text(cell).lower())
+    return unnamed_count >= max(2, len(row) // 3)
+
+
+def _has_second_header_row(rows: list[list[str]]) -> bool:
+    if len(rows) < 2:
+        return False
+    second_row = rows[1]
+    markers = {"姓名", "性别", "生日"}
+    return sum(1 for cell in second_row if clean_text(cell) in markers) >= 2
+
+
+def _combine_headers(main_headers: list[str], sub_headers: list[str]) -> list[str]:
+    result: list[str] = []
+    current_group = ""
+    for index, main in enumerate(main_headers):
+        sub = sub_headers[index] if index < len(sub_headers) else ""
+        main_text = clean_text(main)
+        sub_text = clean_text(sub)
+        if not is_blank(main_text):
+            current_group = main_text
+        elif current_group and sub_text:
+            main_text = current_group
+        if is_blank(main_text):
+            result.append(sub_text)
+            continue
+        if is_blank(sub_text):
+            result.append(main_text)
+            continue
+        if sub_text == main_text:
+            result.append(main_text)
+            continue
+        result.append(f"{main_text}_{sub_text}")
+    return result
+
+
+def _dedupe_headers(headers: list[str]) -> list[str]:
+    seen: dict[str, int] = {}
+    result: list[str] = []
+    for index, header in enumerate(headers):
+        text = clean_text(header)
+        if not text or text.lower().startswith("unnamed"):
+            text = f"column_{index + 1}"
+        count = seen.get(text, 0)
+        seen[text] = count + 1
+        result.append(text if count == 0 else f"{text}_{count + 1}")
+    return result
+
+
+def _looks_like_grouped_record_headers(headers: list[str]) -> bool:
+    grouped_hits = 0
+    normalized_hits = 0
+    for header in headers:
+        if "_" in header:
+            prefix, suffix = header.split("_", 1)
+            if prefix and suffix in {"姓名", "性别", "生日", "出生日期", "手机", "手机号", "电话", "联系电话"}:
+                grouped_hits += 1
+        if resolve_grouped_record_field(header):
+            normalized_hits += 1
+    return grouped_hits >= 2 or normalized_hits >= 3
+
+
+def _rows_to_dicts(rows: list[list[str]], headers: list[str], *, forward_fill: bool) -> list[dict[str, str]]:
+    output: list[dict[str, str]] = []
+    previous_values = {header: "" for header in headers}
+    for raw_row in rows:
+        if not raw_row:
+            continue
+        row: dict[str, str] = {}
+        for index, header in enumerate(headers):
+            value = clean_text(raw_row[index]) if index < len(raw_row) else ""
+            if forward_fill and is_blank(value):
+                value = previous_values.get(header, "")
+            row[header] = value
+            if not is_blank(value):
+                previous_values[header] = value
+        if _is_meaningful_row(row):
+            output.append(row)
+    return output
+
+
+def _is_meaningful_row(row: dict[str, str]) -> bool:
+    non_blank_values = [value for value in row.values() if not is_blank(value)]
+    if not non_blank_values:
+        return False
+    if len(non_blank_values) == 1 and non_blank_values[0].isdigit():
+        return False
+    return True