sap-alv-parser 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sap_alv_parser-0.1.0/PKG-INFO +138 -0
- sap_alv_parser-0.1.0/README.md +120 -0
- sap_alv_parser-0.1.0/pyproject.toml +44 -0
- sap_alv_parser-0.1.0/pyproject.toml.orig +38 -0
- sap_alv_parser-0.1.0/src/sap_alv_parser/__init__.py +5 -0
- sap_alv_parser-0.1.0/src/sap_alv_parser/cli.py +43 -0
- sap_alv_parser-0.1.0/src/sap_alv_parser/parser.py +325 -0
- sap_alv_parser-0.1.0/src/sap_alv_parser/py.typed +0 -0
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: sap-alv-parser
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Parser for SAP ALV fixed-width `|`-delimited reports
|
|
5
|
+
Keywords: sap,alv,parser,fixed-width,text-export
|
|
6
|
+
Author: vccddd
|
|
7
|
+
License-Expression: MIT
|
|
8
|
+
Classifier: Programming Language :: Python :: 3
|
|
9
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
10
|
+
Classifier: Operating System :: OS Independent
|
|
11
|
+
Classifier: Topic :: Text Processing
|
|
12
|
+
Requires-Dist: numpy>=2.5.2
|
|
13
|
+
Requires-Dist: pandas>=3.0.5
|
|
14
|
+
Requires-Dist: prettytable>=3.18.0
|
|
15
|
+
Requires-Python: >=3.13
|
|
16
|
+
Project-URL: Repository, https://github.com/vccddd/sap-alv-parser
|
|
17
|
+
Description-Content-Type: text/markdown
|
|
18
|
+
|
|
19
|
+
# sap-alv-parser
|
|
20
|
+
|
|
21
|
+
Parser for SAP ALV fixed-width `|`-delimited reports. It turns "unconverted" /
|
|
22
|
+
text-format SAP exports into one or more wide tables, merges pagination
|
|
23
|
+
automatically, and hard-codes no column names, column counts, or column widths.
|
|
24
|
+
|
|
25
|
+
## Features
|
|
26
|
+
|
|
27
|
+
- **Zero configuration** — column boundaries, headers, and pagination are all
|
|
28
|
+
inferred automatically; no column names, widths, or counts need to be supplied.
|
|
29
|
+
- **Multiple blocks** — a file's sort-criteria block, data-statistics block, and
|
|
30
|
+
main data table are each parsed into their own table.
|
|
31
|
+
- **CJK friendly** — full-width characters (CJK) are normalized by display width
|
|
32
|
+
(2 columns), so CJK names do not shift column positions.
|
|
33
|
+
- **`|` inside cells** — multi-value fields (e.g. shifts) that use `|` internally
|
|
34
|
+
are not confused with column separators.
|
|
35
|
+
- **Multi-line cells** — cells containing newlines (e.g. multi-line remarks) are
|
|
36
|
+
reassembled correctly, whether the newline falls at the start, middle, or end
|
|
37
|
+
of the cell.
|
|
38
|
+
- **DataFrame-style interface** — every block is a `Table` that can be converted
|
|
39
|
+
to a pandas DataFrame via `to_pandas()`.
|
|
40
|
+
|
|
41
|
+
## Install
|
|
42
|
+
|
|
43
|
+
```bash
|
|
44
|
+
uv add sap-alv-parser # as a dependency
|
|
45
|
+
# or, for local development
|
|
46
|
+
uv sync # installs dependencies (pandas / numpy / pytest)
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
Requirements: Python ≥ 3.13, `pandas`, `numpy`.
|
|
50
|
+
|
|
51
|
+
## Quick start
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
from sap_alv_parser import parse_blocks, parse_table
|
|
55
|
+
|
|
56
|
+
# parse all blocks (each becomes a wide table)
|
|
57
|
+
tables = parse_blocks("report.txt")
|
|
58
|
+
for t in tables:
|
|
59
|
+
print(t.shape, t.columns)
|
|
60
|
+
|
|
61
|
+
# get the main table (the one with the most rows)
|
|
62
|
+
main = parse_table("report.txt")
|
|
63
|
+
df = main.to_pandas() # pandas.DataFrame
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
Command line:
|
|
67
|
+
|
|
68
|
+
```bash
|
|
69
|
+
uv run sap-alv-parse report.txt -o output.csv --all
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
## Recognized grammar
|
|
73
|
+
|
|
74
|
+
These exports are a sequence of *blocks*, each following the same pattern:
|
|
75
|
+
|
|
76
|
+
```
|
|
77
|
+
report := stats_block? page+
|
|
78
|
+
page := sep_line header_line sep_line data_line+
|
|
79
|
+
sep := '-'+ | '|' '-'+ | '|' '-'+ '|'
|
|
80
|
+
header := '|' label ('|' label)*
|
|
81
|
+
data := '|' field ('|' field)* # fixed-width field, may itself contain '|'
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## How it works
|
|
85
|
+
|
|
86
|
+
1. **Single-block parsing** — normalize by display width → infer column boundaries
|
|
87
|
+
from `|` coverage → drop the phantom column created by a trailing edge `|`.
|
|
88
|
+
2. **Multi-block recognition** — a header is the `|` line sandwiched between two
|
|
89
|
+
separator lines and immediately followed by a data line (content-independent);
|
|
90
|
+
blocks sharing the same header (pagination) are merged into one table.
|
|
91
|
+
3. **Record reassembly** — records whose cells contain newlines span several
|
|
92
|
+
physical lines; a record is complete once a `|` appears at every column
|
|
93
|
+
boundary position.
|
|
94
|
+
|
|
95
|
+
## `Table` interface
|
|
96
|
+
|
|
97
|
+
| Capability | Usage | Description |
|
|
98
|
+
|---|---|---|
|
|
99
|
+
| Columns | `t.columns` | `list[str]` |
|
|
100
|
+
| Shape | `t.shape` | `(rows, cols)` |
|
|
101
|
+
| Length | `len(t)` | `int` |
|
|
102
|
+
| Select column | `t['Amount']` | `list[str]` |
|
|
103
|
+
| Select columns | `t[['A','B']]` | sub-`Table` |
|
|
104
|
+
| Head | `t.head(3)` | first 3 rows |
|
|
105
|
+
| Records | `t.to_dict()` | `list[dict]` |
|
|
106
|
+
| Export | `t.to_csv('x.csv')` | CSV file |
|
|
107
|
+
| DataFrame | `t.to_pandas()` | `pandas.DataFrame` |
|
|
108
|
+
|
|
109
|
+
## Scope and limitations
|
|
110
|
+
|
|
111
|
+
Supports SAP ALV grid "unconverted" / text exports in the `|`-delimited
|
|
112
|
+
fixed-width format.
|
|
113
|
+
|
|
114
|
+
**Not automatically covered** (requires separate adapters):
|
|
115
|
+
|
|
116
|
+
- tab-delimited, HTML, or XLSX exports
|
|
117
|
+
- fixed-width text with no `|` delimiters (space-aligned)
|
|
118
|
+
|
|
119
|
+
## Testing
|
|
120
|
+
|
|
121
|
+
> **Note:** the real SAP export files used as test fixtures have been removed
|
|
122
|
+
> from this repository because they contained sensitive production data. All
|
|
123
|
+
> tests now build synthetic fixtures inline and run without any external data.
|
|
124
|
+
|
|
125
|
+
```bash
|
|
126
|
+
uv run pytest
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
## Project structure
|
|
130
|
+
|
|
131
|
+
```
|
|
132
|
+
src/sap_alv_parser/
|
|
133
|
+
├── __init__.py # exports Table / parse_blocks / parse_table
|
|
134
|
+
├── parser.py # core parsing logic
|
|
135
|
+
└── cli.py # sap-alv-parse command line
|
|
136
|
+
tests/
|
|
137
|
+
└── test_parser.py # tests (synthetic fixtures only)
|
|
138
|
+
```
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# sap-alv-parser
|
|
2
|
+
|
|
3
|
+
Parser for SAP ALV fixed-width `|`-delimited reports. It turns "unconverted" /
|
|
4
|
+
text-format SAP exports into one or more wide tables, merges pagination
|
|
5
|
+
automatically, and hard-codes no column names, column counts, or column widths.
|
|
6
|
+
|
|
7
|
+
## Features
|
|
8
|
+
|
|
9
|
+
- **Zero configuration** — column boundaries, headers, and pagination are all
|
|
10
|
+
inferred automatically; no column names, widths, or counts need to be supplied.
|
|
11
|
+
- **Multiple blocks** — a file's sort-criteria block, data-statistics block, and
|
|
12
|
+
main data table are each parsed into their own table.
|
|
13
|
+
- **CJK friendly** — full-width characters (CJK) are normalized by display width
|
|
14
|
+
(2 columns), so CJK names do not shift column positions.
|
|
15
|
+
- **`|` inside cells** — multi-value fields (e.g. shifts) that use `|` internally
|
|
16
|
+
are not confused with column separators.
|
|
17
|
+
- **Multi-line cells** — cells containing newlines (e.g. multi-line remarks) are
|
|
18
|
+
reassembled correctly, whether the newline falls at the start, middle, or end
|
|
19
|
+
of the cell.
|
|
20
|
+
- **DataFrame-style interface** — every block is a `Table` that can be converted
|
|
21
|
+
to a pandas DataFrame via `to_pandas()`.
|
|
22
|
+
|
|
23
|
+
## Install
|
|
24
|
+
|
|
25
|
+
```bash
|
|
26
|
+
uv add sap-alv-parser # as a dependency
|
|
27
|
+
# or, for local development
|
|
28
|
+
uv sync # installs dependencies (pandas / numpy / pytest)
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
Requirements: Python ≥ 3.13, `pandas`, `numpy`.
|
|
32
|
+
|
|
33
|
+
## Quick start
|
|
34
|
+
|
|
35
|
+
```python
|
|
36
|
+
from sap_alv_parser import parse_blocks, parse_table
|
|
37
|
+
|
|
38
|
+
# parse all blocks (each becomes a wide table)
|
|
39
|
+
tables = parse_blocks("report.txt")
|
|
40
|
+
for t in tables:
|
|
41
|
+
print(t.shape, t.columns)
|
|
42
|
+
|
|
43
|
+
# get the main table (the one with the most rows)
|
|
44
|
+
main = parse_table("report.txt")
|
|
45
|
+
df = main.to_pandas() # pandas.DataFrame
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
Command line:
|
|
49
|
+
|
|
50
|
+
```bash
|
|
51
|
+
uv run sap-alv-parse report.txt -o output.csv --all
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
## Recognized grammar
|
|
55
|
+
|
|
56
|
+
These exports are a sequence of *blocks*, each following the same pattern:
|
|
57
|
+
|
|
58
|
+
```
|
|
59
|
+
report := stats_block? page+
|
|
60
|
+
page := sep_line header_line sep_line data_line+
|
|
61
|
+
sep := '-'+ | '|' '-'+ | '|' '-'+ '|'
|
|
62
|
+
header := '|' label ('|' label)*
|
|
63
|
+
data := '|' field ('|' field)* # fixed-width field, may itself contain '|'
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
## How it works
|
|
67
|
+
|
|
68
|
+
1. **Single-block parsing** — normalize by display width → infer column boundaries
|
|
69
|
+
from `|` coverage → drop the phantom column created by a trailing edge `|`.
|
|
70
|
+
2. **Multi-block recognition** — a header is the `|` line sandwiched between two
|
|
71
|
+
separator lines and immediately followed by a data line (content-independent);
|
|
72
|
+
blocks sharing the same header (pagination) are merged into one table.
|
|
73
|
+
3. **Record reassembly** — records whose cells contain newlines span several
|
|
74
|
+
physical lines; a record is complete once a `|` appears at every column
|
|
75
|
+
boundary position.
|
|
76
|
+
|
|
77
|
+
## `Table` interface
|
|
78
|
+
|
|
79
|
+
| Capability | Usage | Description |
|
|
80
|
+
|---|---|---|
|
|
81
|
+
| Columns | `t.columns` | `list[str]` |
|
|
82
|
+
| Shape | `t.shape` | `(rows, cols)` |
|
|
83
|
+
| Length | `len(t)` | `int` |
|
|
84
|
+
| Select column | `t['Amount']` | `list[str]` |
|
|
85
|
+
| Select columns | `t[['A','B']]` | sub-`Table` |
|
|
86
|
+
| Head | `t.head(3)` | first 3 rows |
|
|
87
|
+
| Records | `t.to_dict()` | `list[dict]` |
|
|
88
|
+
| Export | `t.to_csv('x.csv')` | CSV file |
|
|
89
|
+
| DataFrame | `t.to_pandas()` | `pandas.DataFrame` |
|
|
90
|
+
|
|
91
|
+
## Scope and limitations
|
|
92
|
+
|
|
93
|
+
Supports SAP ALV grid "unconverted" / text exports in the `|`-delimited
|
|
94
|
+
fixed-width format.
|
|
95
|
+
|
|
96
|
+
**Not automatically covered** (requires separate adapters):
|
|
97
|
+
|
|
98
|
+
- tab-delimited, HTML, or XLSX exports
|
|
99
|
+
- fixed-width text with no `|` delimiters (space-aligned)
|
|
100
|
+
|
|
101
|
+
## Testing
|
|
102
|
+
|
|
103
|
+
> **Note:** the real SAP export files used as test fixtures have been removed
|
|
104
|
+
> from this repository because they contained sensitive production data. All
|
|
105
|
+
> tests now build synthetic fixtures inline and run without any external data.
|
|
106
|
+
|
|
107
|
+
```bash
|
|
108
|
+
uv run pytest
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
## Project structure
|
|
112
|
+
|
|
113
|
+
```
|
|
114
|
+
src/sap_alv_parser/
|
|
115
|
+
├── __init__.py # exports Table / parse_blocks / parse_table
|
|
116
|
+
├── parser.py # core parsing logic
|
|
117
|
+
└── cli.py # sap-alv-parse command line
|
|
118
|
+
tests/
|
|
119
|
+
└── test_parser.py # tests (synthetic fixtures only)
|
|
120
|
+
```
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sap-alv-parser"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Parser for SAP ALV fixed-width `|`-delimited reports"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
keywords = [
|
|
9
|
+
"sap",
|
|
10
|
+
"alv",
|
|
11
|
+
"parser",
|
|
12
|
+
"fixed-width",
|
|
13
|
+
"text-export",
|
|
14
|
+
]
|
|
15
|
+
classifiers = [
|
|
16
|
+
"Programming Language :: Python :: 3",
|
|
17
|
+
"Programming Language :: Python :: 3.13",
|
|
18
|
+
"Operating System :: OS Independent",
|
|
19
|
+
"Topic :: Text Processing",
|
|
20
|
+
]
|
|
21
|
+
dependencies = [
|
|
22
|
+
"numpy>=2.5.2",
|
|
23
|
+
"pandas>=3.0.5",
|
|
24
|
+
"prettytable>=3.18.0",
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
[[project.authors]]
|
|
28
|
+
name = "vccddd"
|
|
29
|
+
|
|
30
|
+
[project.urls]
|
|
31
|
+
Repository = "https://github.com/vccddd/sap-alv-parser"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
sap-alv-parse = "sap_alv_parser.cli:main"
|
|
35
|
+
|
|
36
|
+
[build-system]
|
|
37
|
+
requires = ["uv_build>=0.11.27,<0.12.0"]
|
|
38
|
+
build-backend = "uv_build"
|
|
39
|
+
|
|
40
|
+
[dependency-groups]
|
|
41
|
+
dev = [
|
|
42
|
+
"pyright>=1.1.411",
|
|
43
|
+
"pytest>=9.1.1",
|
|
44
|
+
]
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "sap-alv-parser"
|
|
3
|
+
version = "0.1.0"
|
|
4
|
+
description = "Parser for SAP ALV fixed-width `|`-delimited reports"
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.13"
|
|
7
|
+
license = "MIT"
|
|
8
|
+
authors = [
|
|
9
|
+
{ name = "vccddd" },
|
|
10
|
+
]
|
|
11
|
+
keywords = ["sap", "alv", "parser", "fixed-width", "text-export"]
|
|
12
|
+
classifiers = [
|
|
13
|
+
"Programming Language :: Python :: 3",
|
|
14
|
+
"Programming Language :: Python :: 3.13",
|
|
15
|
+
"Operating System :: OS Independent",
|
|
16
|
+
"Topic :: Text Processing",
|
|
17
|
+
]
|
|
18
|
+
dependencies = [
|
|
19
|
+
"numpy>=2.5.2",
|
|
20
|
+
"pandas>=3.0.5",
|
|
21
|
+
"prettytable>=3.18.0",
|
|
22
|
+
]
|
|
23
|
+
|
|
24
|
+
[project.urls]
|
|
25
|
+
Repository = "https://github.com/vccddd/sap-alv-parser"
|
|
26
|
+
|
|
27
|
+
[project.scripts]
|
|
28
|
+
sap-alv-parse = "sap_alv_parser.cli:main"
|
|
29
|
+
|
|
30
|
+
[build-system]
|
|
31
|
+
requires = ["uv_build>=0.11.27,<0.12.0"]
|
|
32
|
+
build-backend = "uv_build"
|
|
33
|
+
|
|
34
|
+
[dependency-groups]
|
|
35
|
+
dev = [
|
|
36
|
+
"pyright>=1.1.411",
|
|
37
|
+
"pytest>=9.1.1",
|
|
38
|
+
]
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
"""CLI entry point: parse a SAP export file into multiple wide tables."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import os
|
|
7
|
+
|
|
8
|
+
from .parser import DEFAULT_THRESHOLD, parse_blocks
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def main() -> None:
|
|
12
|
+
ap = argparse.ArgumentParser(
|
|
13
|
+
description="Parse SAP ALV fixed-width `|` reports into multiple blocks"
|
|
14
|
+
)
|
|
15
|
+
ap.add_argument("input", help="input file path")
|
|
16
|
+
ap.add_argument("-o", "--output", default="table.csv", help="CSV path for the largest table")
|
|
17
|
+
ap.add_argument("--threshold", type=float, default=DEFAULT_THRESHOLD,
|
|
18
|
+
help="coverage threshold for column boundary inference (default %(default)s)")
|
|
19
|
+
ap.add_argument("--all", action="store_true", help="write each block to a separate CSV")
|
|
20
|
+
args = ap.parse_args()
|
|
21
|
+
|
|
22
|
+
tables = parse_blocks(args.input, threshold=args.threshold)
|
|
23
|
+
if not tables:
|
|
24
|
+
raise SystemExit("no data blocks parsed.")
|
|
25
|
+
|
|
26
|
+
print(f"Found {len(tables)} block group(s):")
|
|
27
|
+
for i, t in enumerate(tables):
|
|
28
|
+
print(f" [{i}] {t.shape[0]} rows x {t.shape[1]} cols columns: {t.columns}")
|
|
29
|
+
|
|
30
|
+
main = max(tables, key=len)
|
|
31
|
+
main.to_csv(args.output)
|
|
32
|
+
print(f"\nLargest table -> {args.output}: {main.shape[0]} rows x {main.shape[1]} cols")
|
|
33
|
+
|
|
34
|
+
if args.all:
|
|
35
|
+
stem, ext = os.path.splitext(args.output)
|
|
36
|
+
for i, t in enumerate(tables):
|
|
37
|
+
p = f"{stem}.block{i}{ext}"
|
|
38
|
+
t.to_csv(p)
|
|
39
|
+
print(f" block {i} -> {p}")
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
if __name__ == "__main__":
|
|
43
|
+
main()
|
|
@@ -0,0 +1,325 @@
|
|
|
1
|
+
"""Parser for SAP ALV fixed-width `|`-delimited reports, yielding multiple wide tables.
|
|
2
|
+
|
|
3
|
+
The export is a sequence of *blocks*, each following the same pattern::
|
|
4
|
+
|
|
5
|
+
block := sep_line header_line sep_line data_line+
|
|
6
|
+
|
|
7
|
+
sep_line := '-'+ | '|' '-'+ | '|' '-'+ '|'
|
|
8
|
+
header := '|' label ('|' label)*
|
|
9
|
+
data := '|' field ('|' field)* # fixed-width field, may itself contain '|'
|
|
10
|
+
|
|
11
|
+
A file may contain several blocks (sort criteria, data statistics, and the
|
|
12
|
+
paginated main table). Each block is parsed independently; blocks sharing the
|
|
13
|
+
same header (pagination) are merged into a single table.
|
|
14
|
+
|
|
15
|
+
Cells may contain newlines (e.g. multi-line remarks): a record then spans several
|
|
16
|
+
physical lines, and a continuation line may or may not start with `|`. Record
|
|
17
|
+
boundaries are therefore detected by *column-boundary completeness* — a record is
|
|
18
|
+
complete once a `|` appears at every boundary position — rather than by whether a
|
|
19
|
+
line starts with `|`. Control characters (newlines etc.) count as zero-width.
|
|
20
|
+
|
|
21
|
+
Column alignment is auto-detected per block: some exports align by *display width*
|
|
22
|
+
(full-width CJK counts as 2 columns), others by *code points* (CJK counts as 1).
|
|
23
|
+
"""
|
|
24
|
+
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import unicodedata
|
|
28
|
+
from collections.abc import Sequence
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
from typing import TYPE_CHECKING, overload
|
|
31
|
+
|
|
32
|
+
if TYPE_CHECKING:
|
|
33
|
+
import pandas as pd
|
|
34
|
+
|
|
35
|
+
DEFAULT_THRESHOLD: float = 0.95 # coverage threshold for boundary inference
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
class Table:
|
|
39
|
+
"""A lightweight wide table: column names + rows; exports to CSV or pandas.
|
|
40
|
+
|
|
41
|
+
DataFrame-like interface: ``t.columns`` / ``t.shape`` / ``len(t)`` / ``t['col']`` /
|
|
42
|
+
``t[['a','b']]`` / ``t.head()`` / ``t.to_dict()`` / ``t.to_csv()`` / ``t.to_pandas()``.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
def __init__(
|
|
46
|
+
self,
|
|
47
|
+
columns: Sequence[str],
|
|
48
|
+
rows: Sequence[Sequence[str]],
|
|
49
|
+
meta: dict[str, object] | None = None,
|
|
50
|
+
) -> None:
|
|
51
|
+
self.columns: list[str] = list(columns)
|
|
52
|
+
self.rows: list[list[str]] = [list(r) for r in rows]
|
|
53
|
+
self.meta: dict[str, object] = meta or {}
|
|
54
|
+
|
|
55
|
+
def __len__(self) -> int:
|
|
56
|
+
return len(self.rows)
|
|
57
|
+
|
|
58
|
+
def __repr__(self) -> str:
|
|
59
|
+
return f"Table(shape={self.shape}, columns={self.columns})"
|
|
60
|
+
|
|
61
|
+
@property
|
|
62
|
+
def shape(self) -> tuple[int, int]:
|
|
63
|
+
return (len(self.rows), len(self.columns))
|
|
64
|
+
|
|
65
|
+
def head(self, n: int = 5) -> Table:
|
|
66
|
+
return Table(self.columns, self.rows[:n], self.meta)
|
|
67
|
+
|
|
68
|
+
@overload
|
|
69
|
+
def __getitem__(self, key: str) -> list[str]: ...
|
|
70
|
+
|
|
71
|
+
@overload
|
|
72
|
+
def __getitem__(self, key: list[str] | tuple[str, ...]) -> Table: ...
|
|
73
|
+
|
|
74
|
+
def __getitem__(self, key: str | list[str] | tuple[str, ...]) -> list[str] | Table:
|
|
75
|
+
"""Select by column name: ``t['col']`` -> values, ``t[['a','b']]`` -> sub-table."""
|
|
76
|
+
if isinstance(key, str):
|
|
77
|
+
return [r[self.columns.index(key)] for r in self.rows]
|
|
78
|
+
if isinstance(key, (list, tuple)):
|
|
79
|
+
idx = [self.columns.index(k) for k in key]
|
|
80
|
+
return Table(
|
|
81
|
+
[self.columns[i] for i in idx],
|
|
82
|
+
[[r[i] for i in idx] for r in self.rows],
|
|
83
|
+
self.meta,
|
|
84
|
+
)
|
|
85
|
+
raise TypeError("key must be a column name or a list of column names")
|
|
86
|
+
|
|
87
|
+
def to_dict(self) -> list[dict[str, str]]:
|
|
88
|
+
"""Return rows as ``list[dict]`` (records orientation)."""
|
|
89
|
+
return [dict(zip(self.columns, r)) for r in self.rows]
|
|
90
|
+
|
|
91
|
+
def to_csv(self, path: str | Path) -> str:
|
|
92
|
+
import csv
|
|
93
|
+
|
|
94
|
+
with open(path, "w", newline="", encoding="utf-8-sig") as f:
|
|
95
|
+
w = csv.writer(f)
|
|
96
|
+
w.writerow(self.columns)
|
|
97
|
+
w.writerows(self.rows)
|
|
98
|
+
return str(path)
|
|
99
|
+
|
|
100
|
+
def to_pandas(self) -> pd.DataFrame:
|
|
101
|
+
"""Convert to a pandas DataFrame."""
|
|
102
|
+
import pandas as pd
|
|
103
|
+
|
|
104
|
+
return pd.DataFrame(self.rows, columns=self.columns)
|
|
105
|
+
|
|
106
|
+
def to_prettytable(self, replace_pipe: str | None = None, **kwargs):
|
|
107
|
+
"""Convert to a prettytable.PrettyTable.
|
|
108
|
+
|
|
109
|
+
``replace_pipe``, when given, substitutes cell-internal ``|`` (used to
|
|
110
|
+
separate multi-segment values) so it does not visually collide with the
|
|
111
|
+
table's own column separators. Extra ``**kwargs`` go to ``PrettyTable``.
|
|
112
|
+
"""
|
|
113
|
+
import prettytable
|
|
114
|
+
|
|
115
|
+
pt = prettytable.PrettyTable(field_names=self.columns, **kwargs)
|
|
116
|
+
for row in self.rows:
|
|
117
|
+
if replace_pipe is not None:
|
|
118
|
+
row = [c.replace("|", replace_pipe) for c in row]
|
|
119
|
+
pt.add_row(row)
|
|
120
|
+
return pt
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _char_width(c: str, mode: str = "display") -> int:
|
|
125
|
+
if ord(c) < 32: # control characters (\n \r \t) are zero-width
|
|
126
|
+
return 0
|
|
127
|
+
if mode == "display" and unicodedata.east_asian_width(c) in ("F", "W"):
|
|
128
|
+
return 2
|
|
129
|
+
return 1
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def display_width(s: str) -> int:
|
|
133
|
+
return sum(_char_width(c) for c in s)
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def is_separator(line: str) -> bool:
|
|
137
|
+
"""A separator line consists only of '-'/'|' and contains at least one '-'."""
|
|
138
|
+
s = line.strip()
|
|
139
|
+
return bool(s) and ("-" in s) and all(c in "-|" for c in s)
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def header_labels(header_line: str) -> list[str]:
|
|
143
|
+
return [t.strip() for t in header_line.split("|") if t.strip()]
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _pipe_positions(line: str, mode: str = "display") -> list[int]:
|
|
147
|
+
"""Positions of every '|' in a line, in the given width mode (ascending)."""
|
|
148
|
+
positions: list[int] = []
|
|
149
|
+
disp = 0
|
|
150
|
+
for ch in line:
|
|
151
|
+
if ch == "|":
|
|
152
|
+
positions.append(disp)
|
|
153
|
+
disp += _char_width(ch, mode)
|
|
154
|
+
return positions
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _display_to_code(
|
|
159
|
+
line: str, positions: Sequence[int], mode: str = "display"
|
|
160
|
+
) -> dict[int, int]:
|
|
161
|
+
"""Map boundary positions to the nearest '|' code index (±1 tolerance)."""
|
|
162
|
+
mapping: dict[int, int] = {}
|
|
163
|
+
pipes: list[tuple[int, int]] = [] # (position, code index)
|
|
164
|
+
disp = 0
|
|
165
|
+
for j, ch in enumerate(line):
|
|
166
|
+
if ch == "|":
|
|
167
|
+
pipes.append((disp, j))
|
|
168
|
+
disp += _char_width(ch, mode)
|
|
169
|
+
for b in positions:
|
|
170
|
+
best = min(pipes, key=lambda pj: abs(pj[0] - b), default=None)
|
|
171
|
+
if best is not None and abs(best[0] - b) <= 1:
|
|
172
|
+
mapping[b] = best[1]
|
|
173
|
+
return mapping
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def split_row(line: str, boundaries: Sequence[int], mode: str = "display") -> list[str]:
|
|
177
|
+
"""Split one line into cells by boundary positions (stripped)."""
|
|
178
|
+
code = _display_to_code(line, boundaries, mode)
|
|
179
|
+
cols: list[str] = []
|
|
180
|
+
for k in range(len(boundaries) - 1):
|
|
181
|
+
cols.append(line[code[boundaries[k]] + 1 : code[boundaries[k + 1]]].strip())
|
|
182
|
+
cols.append(line[code[boundaries[-1]] + 1 :].strip())
|
|
183
|
+
return cols
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _reassemble(
|
|
187
|
+
data_lines: Sequence[str], boundaries: Sequence[int], mode: str = "display"
|
|
188
|
+
) -> list[str]:
|
|
189
|
+
"""Reassemble physical lines into complete records.
|
|
190
|
+
|
|
191
|
+
A record is complete once a `|` appears near every boundary position.
|
|
192
|
+
A multi-line cell makes a record span several physical lines, and the
|
|
193
|
+
continuation line may or may not start with `|`, so we cannot rely on "line
|
|
194
|
+
starts with `|`" — we rely on boundary completeness instead.
|
|
195
|
+
"""
|
|
196
|
+
boundaries_sorted = sorted(set(boundaries))
|
|
197
|
+
records: list[str] = []
|
|
198
|
+
current: str | None = None
|
|
199
|
+
for line in data_lines:
|
|
200
|
+
current = line if current is None else current + "\n" + line
|
|
201
|
+
positions = set(_pipe_positions(current, mode))
|
|
202
|
+
# every boundary position must have a `|` nearby (±1), tolerating padding drift
|
|
203
|
+
if all(any(abs(b - p) <= 1 for p in positions) for b in boundaries_sorted):
|
|
204
|
+
records.append(current)
|
|
205
|
+
current = None
|
|
206
|
+
if current is not None: # trailing incomplete record (kept as-is)
|
|
207
|
+
records.append(current)
|
|
208
|
+
return records
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _detect_mode(header_line: str, data_lines: Sequence[str]) -> str:
|
|
212
|
+
"""Detect the alignment convention by comparing header vs a data record start.
|
|
213
|
+
|
|
214
|
+
Some exports align columns by display width (CJK counts as 2), others by code
|
|
215
|
+
points (CJK counts as 1). The header and data agree in the correct convention,
|
|
216
|
+
so compare their leading `|` positions in each convention and keep the one
|
|
217
|
+
with more matches. Uses the header (single-line, clean) so it works even when
|
|
218
|
+
every row is a multi-line cell.
|
|
219
|
+
"""
|
|
220
|
+
starts = [l for l in data_lines if l.startswith("|")]
|
|
221
|
+
if not starts:
|
|
222
|
+
return "display"
|
|
223
|
+
ref = starts[0]
|
|
224
|
+
best_mode, best_score = "display", -1
|
|
225
|
+
for mode in ("code", "display"):
|
|
226
|
+
hp = _pipe_positions(header_line, mode)
|
|
227
|
+
dp = _pipe_positions(ref, mode)
|
|
228
|
+
m = min(len(hp), len(dp))
|
|
229
|
+
score = sum(1 for k in range(m) if hp[k] == dp[k])
|
|
230
|
+
if score > best_score:
|
|
231
|
+
best_mode, best_score = mode, score
|
|
232
|
+
return best_mode
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _parse_single_block(
|
|
236
|
+
header_line: str, data_lines: Sequence[str], threshold: float
|
|
237
|
+
) -> Table | None:
|
|
238
|
+
"""Parse a single block (header + its data lines) into a Table."""
|
|
239
|
+
if not data_lines:
|
|
240
|
+
return None
|
|
241
|
+
|
|
242
|
+
mode = _detect_mode(header_line, data_lines)
|
|
243
|
+
boundaries = _pipe_positions(header_line, mode)
|
|
244
|
+
if not boundaries:
|
|
245
|
+
return None
|
|
246
|
+
|
|
247
|
+
records = _reassemble(data_lines, boundaries, mode)
|
|
248
|
+
rows = [split_row(r, boundaries, mode) for r in records]
|
|
249
|
+
|
|
250
|
+
# trailing edge `|` creates a phantom column: drop it if empty in every row
|
|
251
|
+
if rows and all(r[-1] == "" for r in rows):
|
|
252
|
+
boundaries = boundaries[:-1]
|
|
253
|
+
rows = [r[:-1] for r in rows]
|
|
254
|
+
|
|
255
|
+
ncols = len(boundaries)
|
|
256
|
+
labels = header_labels(header_line)
|
|
257
|
+
if len(labels) != ncols:
|
|
258
|
+
labels = [f"col_{i}" for i in range(ncols)]
|
|
259
|
+
return Table(labels, rows)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def split_into_blocks(
|
|
263
|
+
lines: Sequence[str], sep: Sequence[bool]
|
|
264
|
+
) -> list[tuple[str, list[str]]]:
|
|
265
|
+
"""Split a file into blocks, each ``(header_line, data_lines)``."""
|
|
266
|
+
n = len(lines)
|
|
267
|
+
|
|
268
|
+
# A header is a `|` line sandwiched between two separators whose closing
|
|
269
|
+
# separator is immediately followed by a data line. The last condition
|
|
270
|
+
# distinguishes the header from the lone data row of a single-row block.
|
|
271
|
+
header_idx: list[int] = []
|
|
272
|
+
for i in range(1, n - 2):
|
|
273
|
+
l = lines[i]
|
|
274
|
+
if (
|
|
275
|
+
l.startswith("|")
|
|
276
|
+
and not sep[i]
|
|
277
|
+
and sep[i - 1]
|
|
278
|
+
and sep[i + 1]
|
|
279
|
+
and lines[i + 2].startswith("|")
|
|
280
|
+
and not sep[i + 2]
|
|
281
|
+
):
|
|
282
|
+
header_idx.append(i)
|
|
283
|
+
|
|
284
|
+
header_set = set(header_idx)
|
|
285
|
+
blocks: list[tuple[str, list[str]]] = []
|
|
286
|
+
for k, hi in enumerate(header_idx):
|
|
287
|
+
end = header_idx[k + 1] if k + 1 < len(header_idx) else n
|
|
288
|
+
# collect all non-separator, non-blank lines (including multi-line continuations)
|
|
289
|
+
data = [
|
|
290
|
+
lines[j]
|
|
291
|
+
for j in range(hi + 1, end)
|
|
292
|
+
if not sep[j] and lines[j].strip() and j not in header_set
|
|
293
|
+
]
|
|
294
|
+
blocks.append((lines[hi], data))
|
|
295
|
+
return blocks
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def parse_blocks(path: str | Path, threshold: float = DEFAULT_THRESHOLD) -> list[Table]:
|
|
299
|
+
"""Parse all blocks in a file, returning one Table per distinct header (pages merged)."""
|
|
300
|
+
with open(path, "rb") as f:
|
|
301
|
+
raw = f.read().decode("utf-8")
|
|
302
|
+
lines = raw.split("\r\n") if "\r\n" in raw else raw.split("\n")
|
|
303
|
+
sep = [is_separator(l) for l in lines]
|
|
304
|
+
|
|
305
|
+
blocks = split_into_blocks(lines, sep)
|
|
306
|
+
|
|
307
|
+
grouped: dict[tuple[str, ...], Table] = {} # column-name signature -> Table
|
|
308
|
+
for header_line, data_lines in blocks:
|
|
309
|
+
t = _parse_single_block(header_line, data_lines, threshold)
|
|
310
|
+
if t is None:
|
|
311
|
+
continue
|
|
312
|
+
key = tuple(t.columns)
|
|
313
|
+
if key in grouped:
|
|
314
|
+
grouped[key].rows.extend(t.rows)
|
|
315
|
+
else:
|
|
316
|
+
grouped[key] = t
|
|
317
|
+
return list(grouped.values())
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def parse_table(path: str | Path, threshold: float = DEFAULT_THRESHOLD) -> Table:
|
|
321
|
+
"""Return the table with the most rows (usually the main data table)."""
|
|
322
|
+
tables = parse_blocks(path, threshold)
|
|
323
|
+
if not tables:
|
|
324
|
+
raise ValueError(f"no data blocks parsed from {path}")
|
|
325
|
+
return max(tables, key=len)
|
|
File without changes
|