excel-list-transform 0.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- excel_list_transform/__init__.py +1 -0
- excel_list_transform/__main__.py +11 -0
- excel_list_transform/check_indata_common.py +39 -0
- excel_list_transform/commontypes.py +59 -0
- excel_list_transform/config.py +358 -0
- excel_list_transform/config_enums.py +53 -0
- excel_list_transform/config_excel_list_transform.py +278 -0
- excel_list_transform/config_factory.py +95 -0
- excel_list_transform/config_xls_list_refmt_name.py +40 -0
- excel_list_transform/config_xls_list_refmt_num.py +70 -0
- excel_list_transform/file_extension.py +30 -0
- excel_list_transform/file_must_exist.py +22 -0
- excel_list_transform/generate_cfg.py +374 -0
- excel_list_transform/generate_txt.py +259 -0
- excel_list_transform/handle_csv.py +73 -0
- excel_list_transform/handle_empty_column.py +28 -0
- excel_list_transform/handle_excel.py +161 -0
- excel_list_transform/num_named_conversion.py +60 -0
- excel_list_transform/rewrite_value.py +123 -0
- excel_list_transform/str_to_enum.py +39 -0
- excel_list_transform/transform_cmd.py +146 -0
- excel_list_transform/transform_func.py +50 -0
- excel_list_transform/transform_func_common.py +249 -0
- excel_list_transform/transform_func_named.py +136 -0
- excel_list_transform/transform_func_num.py +157 -0
- excel_list_transform-0.5.dist-info/LICENSE.txt +22 -0
- excel_list_transform-0.5.dist-info/METADATA +114 -0
- excel_list_transform-0.5.dist-info/RECORD +30 -0
- excel_list_transform-0.5.dist-info/WHEEL +5 -0
- excel_list_transform-0.5.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Handle reading and writing of an excel file."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from csv import reader, DictReader, writer, DictWriter, Dialect
|
|
9
|
+
from excel_list_transform.handle_empty_column import \
|
|
10
|
+
handle_empty_column_in_lst, handle_empty_column_in_dict_lst
|
|
11
|
+
from excel_list_transform.num_named_conversion import named_cols_from_num_cols
|
|
12
|
+
from excel_list_transform.commontypes import NumData, NumDataSeq, NameData
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def read_csv_num(filename: str, dialect: type[Dialect],
|
|
16
|
+
max_column_read: int) -> NumData:
|
|
17
|
+
"""Read a csv file in specified dialect (numbered columns)."""
|
|
18
|
+
res: NumDataSeq = []
|
|
19
|
+
with open(file=filename, mode="r",
|
|
20
|
+
encoding='utf_8_sig', newline='') as cfile:
|
|
21
|
+
creader = reader(cfile, dialect=dialect)
|
|
22
|
+
for row in creader:
|
|
23
|
+
res.append(list(row)[:max_column_read])
|
|
24
|
+
return handle_empty_column_in_lst(res)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_csv_named_use_num(filename: str, dialect: type[Dialect],
|
|
28
|
+
max_column_read: int) -> NameData:
|
|
29
|
+
"""Read a csv file using numbered columns for named columns."""
|
|
30
|
+
data = read_csv_num(filename=filename, dialect=dialect,
|
|
31
|
+
max_column_read=max_column_read)
|
|
32
|
+
return named_cols_from_num_cols(data=data, filename=filename)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def read_csv_named(filename: str, dialect: type[Dialect],
|
|
36
|
+
max_column_read: int) -> NameData:
|
|
37
|
+
"""Read a csv file in specified dialect (named columns)."""
|
|
38
|
+
res = []
|
|
39
|
+
first_line: bool = True
|
|
40
|
+
with open(file=filename, mode="r",
|
|
41
|
+
encoding='utf_8_sig', newline='') as cfile:
|
|
42
|
+
creader = DictReader(cfile, dialect=dialect)
|
|
43
|
+
for row in creader:
|
|
44
|
+
if first_line:
|
|
45
|
+
if len(row) > max_column_read:
|
|
46
|
+
return read_csv_named_use_num(filename=filename,
|
|
47
|
+
dialect=dialect,
|
|
48
|
+
max_column_read= # noqa: E251, E501
|
|
49
|
+
max_column_read)
|
|
50
|
+
first_line = False
|
|
51
|
+
res.append(row)
|
|
52
|
+
return handle_empty_column_in_dict_lst(res)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def write_csv_num(data: NumData,
|
|
56
|
+
filename: str, dialect: type[Dialect]) -> None:
|
|
57
|
+
"""Write a csv file in specified dialect."""
|
|
58
|
+
with open(file=filename, mode="w", encoding='utf8', newline='') as cfile:
|
|
59
|
+
cwriter = writer(cfile, dialect=dialect)
|
|
60
|
+
for row in data:
|
|
61
|
+
cwriter.writerow(row)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def write_csv_named(data: NameData,
|
|
65
|
+
filename: str, dialect: type[Dialect],
|
|
66
|
+
column_order: list[str]) -> None:
|
|
67
|
+
"""Write a csv file in specified dialect."""
|
|
68
|
+
with open(file=filename, mode="w", encoding='utf8', newline='') as cfile:
|
|
69
|
+
cwriter = DictWriter(f=cfile, dialect=dialect, fieldnames=column_order,
|
|
70
|
+
extrasaction='ignore')
|
|
71
|
+
cwriter.writeheader()
|
|
72
|
+
for row in data:
|
|
73
|
+
cwriter.writerow(row)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Handle empty column in input."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from excel_list_transform.commontypes import NameData, NameRow, \
|
|
9
|
+
NumData, NumRow, NumDataSeq, NameDataMap
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def handle_empty_column_in_lst(input_data: NumData | NumDataSeq) -> NumData:
|
|
13
|
+
"""Handle empty column in list of lists of input data."""
|
|
14
|
+
ret: NumData = []
|
|
15
|
+
for row in input_data:
|
|
16
|
+
nrow: NumRow = [i if i != '' else None for i in row]
|
|
17
|
+
ret.append(nrow)
|
|
18
|
+
return ret
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def handle_empty_column_in_dict_lst(input_data: NameData | NameDataMap) \
|
|
22
|
+
-> NameData:
|
|
23
|
+
"""Handle empty column in list of lists of input data."""
|
|
24
|
+
ret = []
|
|
25
|
+
for row in input_data:
|
|
26
|
+
nrow: NameRow = {k: (v if v != '' else None) for (k, v) in row.items()}
|
|
27
|
+
ret.append(nrow)
|
|
28
|
+
return ret
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
#! /usr/bin/python3
|
|
2
|
+
"""Handle reading and writing of an excel file."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
# https://realpython.com/openpyxl-excel-spreadsheets-python/
|
|
9
|
+
|
|
10
|
+
import sys
|
|
11
|
+
from typing import Optional
|
|
12
|
+
from openpyxl import load_workbook as openpyxl_load_workbook
|
|
13
|
+
from openpyxl import Workbook as openpyxl_wb
|
|
14
|
+
from xlsxwriter import Workbook as xlsxwriter_wb # type: ignore
|
|
15
|
+
from pylightxl import readxl as pl_readxl # type: ignore
|
|
16
|
+
from pylightxl import Database as pl_db, writexl as pl_writexl
|
|
17
|
+
from excel_list_transform.commontypes import NameData, NameDataMap, \
|
|
18
|
+
NumData, NumDataSeq
|
|
19
|
+
from excel_list_transform.config_enums import ExcelLib
|
|
20
|
+
from excel_list_transform.handle_empty_column import handle_empty_column_in_lst
|
|
21
|
+
from excel_list_transform.num_named_conversion import \
|
|
22
|
+
named_cols_from_num_cols, num_cols_from_named_cols
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def read_excel_openpyxl(filename: str,
|
|
26
|
+
max_column_read: int) \
|
|
27
|
+
-> NumData:
|
|
28
|
+
"""Read the excel file using openpyxl."""
|
|
29
|
+
workbook = openpyxl_load_workbook(filename=filename, read_only=True,
|
|
30
|
+
data_only=True)
|
|
31
|
+
sheet = workbook.active
|
|
32
|
+
assert sheet is not None
|
|
33
|
+
res: NumData = []
|
|
34
|
+
for row in sheet.iter_rows(values_only=True):
|
|
35
|
+
res.append(list(row)[:max_column_read])
|
|
36
|
+
return handle_empty_column_in_lst(res)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def read_excel_pyligthxl(filename: str,
|
|
40
|
+
max_column_read: int) \
|
|
41
|
+
-> NumData:
|
|
42
|
+
"""Read the excel file using pyligthxl."""
|
|
43
|
+
pyl_db = pl_readxl(fn=filename)
|
|
44
|
+
sheetnames = pyl_db.ws_names
|
|
45
|
+
res = []
|
|
46
|
+
for sheet in sheetnames:
|
|
47
|
+
for row in pyl_db.ws(ws=sheet).rows:
|
|
48
|
+
res.append(list(row)[:max_column_read])
|
|
49
|
+
ret = []
|
|
50
|
+
for row in res:
|
|
51
|
+
row = [i if i != '' else None for i in row]
|
|
52
|
+
ret.append(row)
|
|
53
|
+
return ret
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def read_excel_xlsxwriter(filename: str,
|
|
57
|
+
max_column_read: int) \
|
|
58
|
+
-> NumData:
|
|
59
|
+
"""Flag error and use other read library."""
|
|
60
|
+
print('xlsxwriter cannot read, using pylightxl.', file=sys.stderr)
|
|
61
|
+
return read_excel_pyligthxl(filename=filename,
|
|
62
|
+
max_column_read=max_column_read)
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def read_excel_num(filename: str, max_column_read: int,
|
|
66
|
+
excel_lib: Optional[ExcelLib] = None) \
|
|
67
|
+
-> NumData:
|
|
68
|
+
"""Read the excel file for number referenced columns."""
|
|
69
|
+
dispatch = {ExcelLib.OPENPYXL: read_excel_openpyxl,
|
|
70
|
+
ExcelLib.PYLIGHTXL: read_excel_pyligthxl,
|
|
71
|
+
ExcelLib.XLSXWRITER: read_excel_xlsxwriter}
|
|
72
|
+
if excel_lib is None:
|
|
73
|
+
excel_lib = ExcelLib.PYLIGHTXL # default for now
|
|
74
|
+
return dispatch[excel_lib](filename=filename,
|
|
75
|
+
max_column_read=max_column_read)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def read_excel_named(filename: str, max_column_read: int,
|
|
79
|
+
excel_lib: Optional[ExcelLib] = None) \
|
|
80
|
+
-> NameData:
|
|
81
|
+
"""Read the excel file for name referenced columns."""
|
|
82
|
+
data = read_excel_num(filename=filename, max_column_read=max_column_read,
|
|
83
|
+
excel_lib=excel_lib)
|
|
84
|
+
return named_cols_from_num_cols(data=data, filename=filename)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def write_excel_openpyxl(data: NumData | NumDataSeq,
|
|
88
|
+
filename: str) -> None:
|
|
89
|
+
"""Write data as list in excel file using openpyxl."""
|
|
90
|
+
workbook = openpyxl_wb()
|
|
91
|
+
sheet = workbook.active
|
|
92
|
+
assert sheet is not None
|
|
93
|
+
workbook.encoding = 'utf-8'
|
|
94
|
+
assert isinstance(data, list)
|
|
95
|
+
for row in data:
|
|
96
|
+
assert isinstance(row, (list, tuple))
|
|
97
|
+
sheet.append(row)
|
|
98
|
+
workbook.save(filename=filename)
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def write_excel_pylightxl(data: NumData | NumDataSeq,
|
|
102
|
+
filename: str) -> None:
|
|
103
|
+
"""Write data as list in excel file using pylightxl."""
|
|
104
|
+
pyl_db = pl_db()
|
|
105
|
+
pyl_db.add_ws(ws="Data")
|
|
106
|
+
for row_id, row in enumerate(data, start=1):
|
|
107
|
+
for col_id, elem in enumerate(row, start=1):
|
|
108
|
+
value = elem if elem is not None else ''
|
|
109
|
+
pyl_db.ws(ws="Data").update_index(row=row_id, col=col_id,
|
|
110
|
+
val=value)
|
|
111
|
+
pl_writexl(db=pyl_db, fn=filename)
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def write_excel_xlsxwriter(data: NumData | NumDataSeq,
|
|
115
|
+
filename: str) -> None:
|
|
116
|
+
"""Write data as list in excel file using xlsxwriter."""
|
|
117
|
+
options = {'in_memory': True,
|
|
118
|
+
'strings_to_numbers': False,
|
|
119
|
+
'strings_to_formulaes': False,
|
|
120
|
+
'strings_to_urls': False,
|
|
121
|
+
'default_date_format': 'yyyy-mm-dd',
|
|
122
|
+
'remove_timezone': True}
|
|
123
|
+
with xlsxwriter_wb(filename=filename, options=options) as workbook:
|
|
124
|
+
sheet = workbook.add_worksheet()
|
|
125
|
+
assert isinstance(data, list)
|
|
126
|
+
for i, row in enumerate(data):
|
|
127
|
+
assert isinstance(row, (list, tuple))
|
|
128
|
+
for j, elem in enumerate(row):
|
|
129
|
+
if elem is not None:
|
|
130
|
+
if isinstance(elem, str):
|
|
131
|
+
sheet.write_string(row=i, col=j, string=elem)
|
|
132
|
+
elif isinstance(elem, int):
|
|
133
|
+
sheet.write_number(row=i, col=j, number=elem)
|
|
134
|
+
else:
|
|
135
|
+
msg = f'Unexpected data type {type(elem).__name__} '
|
|
136
|
+
msg += f'for row={i} col={j}. Value {elem}.'
|
|
137
|
+
print(msg, file=sys.stderr)
|
|
138
|
+
raise RuntimeError(msg)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def write_excel_num(data: NumData | NumDataSeq,
|
|
142
|
+
filename: str,
|
|
143
|
+
excel_lib: Optional[ExcelLib] = None) -> None:
|
|
144
|
+
"""Write data (number referenced columns) as list in excel file."""
|
|
145
|
+
dispatch = {ExcelLib.OPENPYXL: write_excel_openpyxl,
|
|
146
|
+
ExcelLib.PYLIGHTXL: write_excel_pylightxl,
|
|
147
|
+
ExcelLib.XLSXWRITER: write_excel_xlsxwriter}
|
|
148
|
+
if excel_lib is None:
|
|
149
|
+
excel_lib = ExcelLib.XLSXWRITER # default for now
|
|
150
|
+
dispatch[excel_lib](data=data, filename=filename)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def write_excel_named(data: NameDataMap,
|
|
154
|
+
filename: str,
|
|
155
|
+
column_order: list[str],
|
|
156
|
+
excel_lib: Optional[ExcelLib] = None) -> None:
|
|
157
|
+
"""Write data (name referenced columns) as list in excel file."""
|
|
158
|
+
wdata: NumData = num_cols_from_named_cols(data=data,
|
|
159
|
+
column_order=column_order)
|
|
160
|
+
write_excel_num(data=wdata, filename=filename,
|
|
161
|
+
excel_lib=excel_lib)
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Convert data between numbered columns and named columns."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
import sys
|
|
9
|
+
from excel_list_transform.commontypes import NameData, NumData, \
|
|
10
|
+
NumDataSeq, NumRow, NameDataMap, \
|
|
11
|
+
num_row_to_str_list, str_list_to_num_row
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def named_cols_from_num_cols(data: NumData | NumDataSeq,
|
|
15
|
+
filename: str) -> NameData:
|
|
16
|
+
"""Convert data with numbered columns to data with named columns.
|
|
17
|
+
|
|
18
|
+
If rows have too few columns, pad as needed with None.
|
|
19
|
+
"""
|
|
20
|
+
filemsg = f' in file {filename}.'
|
|
21
|
+
for i, name in enumerate(data[0]):
|
|
22
|
+
if name is None:
|
|
23
|
+
print(f'Cannot handle input column {i} without name' + filemsg,
|
|
24
|
+
file=sys.stderr)
|
|
25
|
+
sys.exit(1)
|
|
26
|
+
names: list[str] = num_row_to_str_list(data[0])
|
|
27
|
+
num_cols = len(names)
|
|
28
|
+
ret: NameData = []
|
|
29
|
+
for row_in in data[1:]:
|
|
30
|
+
nrow: NumRow = list(row_in)
|
|
31
|
+
if len(nrow) > num_cols:
|
|
32
|
+
print('Data row(s) have more columns than title row' + filemsg,
|
|
33
|
+
file=sys.stderr)
|
|
34
|
+
sys.exit(1)
|
|
35
|
+
while len(nrow) < num_cols:
|
|
36
|
+
nrow.append(None)
|
|
37
|
+
ret.append(dict(zip(names, nrow)))
|
|
38
|
+
return ret
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def num_cols_from_named_cols(data: NameData | NameDataMap,
|
|
42
|
+
column_order: list[str]) -> NumData:
|
|
43
|
+
"""Convert data with named columns to data with numbered columns.
|
|
44
|
+
|
|
45
|
+
Silently ignore data with keys not in column_order.
|
|
46
|
+
If key is missing flag it as error.
|
|
47
|
+
"""
|
|
48
|
+
ret: NumData = []
|
|
49
|
+
ret.append(str_list_to_num_row(column_order))
|
|
50
|
+
for rownum, row in enumerate(data):
|
|
51
|
+
outrow: NumRow = []
|
|
52
|
+
for key in column_order:
|
|
53
|
+
if key in row:
|
|
54
|
+
outrow.append(row[key])
|
|
55
|
+
else:
|
|
56
|
+
msg = f'Data row {rownum} is missing data for column {key}'
|
|
57
|
+
print(msg, file=sys.stderr)
|
|
58
|
+
sys.exit(1)
|
|
59
|
+
ret.append(outrow)
|
|
60
|
+
return ret
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Functions for manipulating and rewriting a value."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from copy import deepcopy
|
|
9
|
+
import sys
|
|
10
|
+
from typing import Optional, assert_never
|
|
11
|
+
from re import compile as re_compile
|
|
12
|
+
from re import Pattern
|
|
13
|
+
from re import IGNORECASE as re_IGNORECASE
|
|
14
|
+
from re import error as re_error
|
|
15
|
+
from excel_list_transform.commontypes import get_checked_type
|
|
16
|
+
from excel_list_transform.config_enums import RewriteKind, CaseSensitivity
|
|
17
|
+
from excel_list_transform.config_excel_list_transform import \
|
|
18
|
+
Column, SingleRuleRewrite
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def strip_value(value: str, chars: str | None,
|
|
22
|
+
casehandle: CaseSensitivity) -> str:
|
|
23
|
+
"""Strip off characters from beginning/end of value."""
|
|
24
|
+
if chars is None or chars == '':
|
|
25
|
+
return value.strip()
|
|
26
|
+
cha = chars
|
|
27
|
+
if casehandle == CaseSensitivity.IGNORE_CASE and len(chars) > 0:
|
|
28
|
+
cha = chars.lower() + chars.upper()
|
|
29
|
+
return value.strip(cha)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def str_replace_value(value: str, fro: str, to: str,
|
|
33
|
+
casehandle: CaseSensitivity) -> str:
|
|
34
|
+
"""Replace sub-strings in value."""
|
|
35
|
+
if casehandle == CaseSensitivity.MATCH_CASE:
|
|
36
|
+
return value.replace(fro, to)
|
|
37
|
+
fromstr = deepcopy(fro)
|
|
38
|
+
pos = -1
|
|
39
|
+
while True:
|
|
40
|
+
ret = deepcopy(value)
|
|
41
|
+
retl = ret.lower()
|
|
42
|
+
froml = fromstr.lower()
|
|
43
|
+
pos = retl.find(froml, pos+1)
|
|
44
|
+
if pos < 0: # from not found any more
|
|
45
|
+
break
|
|
46
|
+
fro_cased = value[pos:pos+len(fro)]
|
|
47
|
+
value = value.replace(fro_cased, to)
|
|
48
|
+
return value
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def remove_from_value(value: str, chars: list[str],
|
|
52
|
+
casehandle: CaseSensitivity) -> str:
|
|
53
|
+
"""Remove sub-string from value."""
|
|
54
|
+
if len(chars) == 0:
|
|
55
|
+
return value
|
|
56
|
+
for cha in chars:
|
|
57
|
+
value = str_replace_value(value=value, fro=cha, to='',
|
|
58
|
+
casehandle=casehandle)
|
|
59
|
+
return value
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def regex_replace_value(value: str, fro: str, to: str,
|
|
63
|
+
casehandle: CaseSensitivity) -> str:
|
|
64
|
+
"""Replace regexp match in value."""
|
|
65
|
+
flags = re_IGNORECASE if casehandle == CaseSensitivity.IGNORE_CASE else 0
|
|
66
|
+
if fro in regex_replace_value_cache:
|
|
67
|
+
cached: dict[int, Pattern[str]] = regex_replace_value_cache[fro]
|
|
68
|
+
if flags in cached:
|
|
69
|
+
re_obj: Pattern[str] = cached[flags]
|
|
70
|
+
assert re_obj is not None
|
|
71
|
+
return re_obj.sub(repl=to, string=value)
|
|
72
|
+
try:
|
|
73
|
+
re_obj2: Pattern[str] = re_compile(pattern=fro, flags=flags)
|
|
74
|
+
except re_error as exc:
|
|
75
|
+
msg = 's7_rewrite_columns: invalid regular expression (regex): '
|
|
76
|
+
msg += f'"{fro}"\n'
|
|
77
|
+
msg += str(exc)
|
|
78
|
+
print(msg, file=sys.stderr)
|
|
79
|
+
sys.exit(1)
|
|
80
|
+
assert re_obj2 is not None
|
|
81
|
+
if fro in regex_replace_value_cache:
|
|
82
|
+
regex_replace_value_cache[fro][flags] = re_obj2
|
|
83
|
+
else:
|
|
84
|
+
regex_replace_value_cache[fro] = {flags: re_obj2}
|
|
85
|
+
return re_obj2.sub(repl=to, string=value)
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
regex_replace_value_cache: dict[str, dict[int, Pattern[str]]] = {}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def rewrite_value(value: Optional[str],
|
|
92
|
+
spec: SingleRuleRewrite[Column],
|
|
93
|
+
tinfo: Column) -> Optional[str]:
|
|
94
|
+
"""Return a value rewritten according to spec."""
|
|
95
|
+
if value is None:
|
|
96
|
+
return None
|
|
97
|
+
assert value is not None
|
|
98
|
+
assert tinfo is not None # tinfo needed for mypy, avoid pylint unused
|
|
99
|
+
ret: str = deepcopy(value)
|
|
100
|
+
kind: RewriteKind = get_checked_type(spec['kind'], RewriteKind)
|
|
101
|
+
casehandle: CaseSensitivity = get_checked_type(spec['case'],
|
|
102
|
+
CaseSensitivity)
|
|
103
|
+
if kind == RewriteKind.STRIP:
|
|
104
|
+
chars1 = spec['chars']
|
|
105
|
+
if chars1 is not None:
|
|
106
|
+
assert isinstance(chars1, str)
|
|
107
|
+
return strip_value(value=ret, chars=chars1,
|
|
108
|
+
casehandle=casehandle)
|
|
109
|
+
if kind == RewriteKind.REMOVECHARS:
|
|
110
|
+
chars: list[str] = get_checked_type(spec['chars'], list)
|
|
111
|
+
return remove_from_value(value=ret, chars=chars,
|
|
112
|
+
casehandle=casehandle)
|
|
113
|
+
fro: str = get_checked_type(spec['from'], str)
|
|
114
|
+
to: str = get_checked_type(spec['to'], str)
|
|
115
|
+
if kind == RewriteKind.STR_SUBSTITUTE:
|
|
116
|
+
return str_replace_value(value=ret, fro=fro, to=to,
|
|
117
|
+
casehandle=casehandle)
|
|
118
|
+
if kind == RewriteKind.REGEX_SUBSTITUTE:
|
|
119
|
+
return regex_replace_value(value=ret, fro=fro, to=to,
|
|
120
|
+
casehandle=casehandle)
|
|
121
|
+
assert_never(f'unexpected value kind={kind.name} in ' +
|
|
122
|
+
'rewrite_value') # pylint: disable=unreachable,line-too-long # pragma: no cover # noqa: E501
|
|
123
|
+
assert False # pylint: disable=unreachable,line-too-long # pragma: no cover # noqa: E501
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Functions to fuzzy parse string into enum."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from enum import Enum
|
|
9
|
+
from typing import TypeVar, Optional
|
|
10
|
+
|
|
11
|
+
SomeEnum = TypeVar('SomeEnum', bound=Enum)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def string_to_enum_best_match(inp: str, num_type: type[SomeEnum]) -> SomeEnum:
|
|
15
|
+
"""Find the enum that is the best match for string.
|
|
16
|
+
|
|
17
|
+
Parameters:
|
|
18
|
+
inp: the string that should match an enum value
|
|
19
|
+
num_type: the enumeration class to find enum values in
|
|
20
|
+
"""
|
|
21
|
+
assert isinstance(inp, str), 'string_to_enum_best_match called ' + \
|
|
22
|
+
f'with {type(inp).__name__} not str as expected.'
|
|
23
|
+
for variant in (inp, inp.capitalize(), inp.lower(), inp.upper()):
|
|
24
|
+
try:
|
|
25
|
+
return num_type[variant]
|
|
26
|
+
except KeyError:
|
|
27
|
+
pass
|
|
28
|
+
num_match: int = 0
|
|
29
|
+
match: Optional[SomeEnum] = None
|
|
30
|
+
for i in num_type:
|
|
31
|
+
if i.name.upper()[0:len(inp)] == inp.upper():
|
|
32
|
+
num_match += 1
|
|
33
|
+
match = i
|
|
34
|
+
if num_match == 1 and match is not None:
|
|
35
|
+
assert match is not None
|
|
36
|
+
assert isinstance(match, num_type)
|
|
37
|
+
return match
|
|
38
|
+
errstr = inp + ' is not one of: ' + ', '.join([e.name for e in num_type])
|
|
39
|
+
raise KeyError(errstr)
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Command to transform list in excel or CSV file."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from copy import deepcopy
|
|
9
|
+
from sys import argv as sys_argv
|
|
10
|
+
from typing import Optional, TypeAlias
|
|
11
|
+
import argparse
|
|
12
|
+
from excel_list_transform.transform_func import transform_named_files
|
|
13
|
+
from excel_list_transform.generate_cfg import generate_examplecfg
|
|
14
|
+
from excel_list_transform.generate_cfg import get_example_names
|
|
15
|
+
from excel_list_transform.config_enums import ColumnRef
|
|
16
|
+
from excel_list_transform.str_to_enum import string_to_enum_best_match
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def gen_cfg_cmd(args: argparse.Namespace) -> int:
|
|
20
|
+
"""Generate example cfg file."""
|
|
21
|
+
outfilename: str = args.output[0]
|
|
22
|
+
cfgtype: str = args.kind[0]
|
|
23
|
+
colref: ColumnRef = string_to_enum_best_match(args.reference[0],
|
|
24
|
+
ColumnRef)
|
|
25
|
+
return generate_examplecfg(filename=outfilename,
|
|
26
|
+
cfgtype=cfgtype, colref=colref)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def rfmt_cmd(args: argparse.Namespace) -> int:
|
|
30
|
+
"""Generate example cfg file."""
|
|
31
|
+
outfilename = args.output[0]
|
|
32
|
+
infilename = args.input[0]
|
|
33
|
+
cfgfilename = args.cfg[0]
|
|
34
|
+
return transform_named_files(infilename=infilename,
|
|
35
|
+
outfilename=outfilename,
|
|
36
|
+
cfgfilename=cfgfilename)
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
USAGE_ORDER = '''
|
|
40
|
+
The normal way to use this command is:
|
|
41
|
+
(1) Using the "example" sub-command a few example configuration (.cfg) files
|
|
42
|
+
with description (.txt) files are generated.
|
|
43
|
+
(2) Read the example configuration (.cfg) files and the accompanying
|
|
44
|
+
description (.txt) files.
|
|
45
|
+
(3) Find an example that is close to what you want to achieve.
|
|
46
|
+
(4) Modify that configuration file to achieve what you want to achieve.
|
|
47
|
+
(5) Use the "transform" sub-command to read your data and output it
|
|
48
|
+
transformed or reorganized according to your modified configuration
|
|
49
|
+
file.
|
|
50
|
+
(6) Read the produced output. If necessary go back to step 4 and adjust
|
|
51
|
+
how the data is transformed.
|
|
52
|
+
'''
|
|
53
|
+
|
|
54
|
+
TXT_DESCRIPTION = '''
|
|
55
|
+
When generating an example configuration file a text file describing
|
|
56
|
+
the configuration file syntax is also generated, with the same name
|
|
57
|
+
as the configuration file but with extension .txt instead of .cfg.
|
|
58
|
+
'''
|
|
59
|
+
|
|
60
|
+
GENERAL_DESCRIPTION = '''
|
|
61
|
+
Transform or reorganize list in excel or CSV file. How data is transformed
|
|
62
|
+
is described in a configuration file. Name of input file,
|
|
63
|
+
output file and configuration file is given as command line
|
|
64
|
+
arguments.
|
|
65
|
+
The command can also generate a few example configuration files.
|
|
66
|
+
When generating an example configuration file the output file name
|
|
67
|
+
switch gives the name of the generated configuration file.
|
|
68
|
+
'''
|
|
69
|
+
|
|
70
|
+
SEE_MAIN_HELP = '''
|
|
71
|
+
See also help text for main command without sub-commands.
|
|
72
|
+
'''
|
|
73
|
+
|
|
74
|
+
SubParseAct: TypeAlias = 'argparse._SubParsersAction[argparse.ArgumentParser]'
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def gen_cfg_args(subparsers: SubParseAct) -> None:
|
|
78
|
+
"""Add arguments for generate example config sub-command."""
|
|
79
|
+
cfg_help = 'Generate example configuration file (example .cfg file). '
|
|
80
|
+
cfg_help += 'Arguments select the kind of configuration file that '
|
|
81
|
+
cfg_help += 'is generated.'
|
|
82
|
+
cfg_parser = subparsers.add_parser('example', help=cfg_help,
|
|
83
|
+
epilog=USAGE_ORDER,
|
|
84
|
+
description=cfg_help +
|
|
85
|
+
TXT_DESCRIPTION + SEE_MAIN_HELP)
|
|
86
|
+
cfg_parser.set_defaults(func=gen_cfg_cmd)
|
|
87
|
+
examplekinds = get_example_names()
|
|
88
|
+
kind_help = 'Kind of example to generate configuration file for.'
|
|
89
|
+
kind_help += 'Possible kinds are (' + ', '.join(examplekinds) + ').'
|
|
90
|
+
cfg_parser.add_argument('-k', '--kind', nargs=1, required=True,
|
|
91
|
+
help=kind_help, choices=examplekinds)
|
|
92
|
+
reftypes = [e.name.lower() for e in ColumnRef]
|
|
93
|
+
ref_help = 'How does configuration file reference to columns '
|
|
94
|
+
ref_help += 'in the data. By column number of by column name. '
|
|
95
|
+
ref_help += 'Possible values are (' + ', '.join(reftypes) + '). '
|
|
96
|
+
ref_help += 'Normally "by_name" column references are easier to get right.'
|
|
97
|
+
cfg_parser.add_argument('-r', '--reference', nargs=1, required=True,
|
|
98
|
+
help=ref_help, choices=reftypes)
|
|
99
|
+
cfg_output_help = 'Name of configuration (output) file to create.'
|
|
100
|
+
cfg_parser.add_argument('-o', '--output', nargs=1,
|
|
101
|
+
help=cfg_output_help, required=True)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def rfmt_args(subparsers: SubParseAct) -> None:
|
|
105
|
+
"""Add arguments for transform sub-command."""
|
|
106
|
+
rfmt_help = 'Transform list in excel or CSV file. How data is transformed '
|
|
107
|
+
rfmt_help += 'is described in a configuration file. Name of input file, '
|
|
108
|
+
rfmt_help += 'output file and configuration file is given as command '
|
|
109
|
+
rfmt_help += 'line arguments.'
|
|
110
|
+
rfmt_parser = subparsers.add_parser('transform', help=rfmt_help,
|
|
111
|
+
epilog=USAGE_ORDER,
|
|
112
|
+
description=rfmt_help + SEE_MAIN_HELP)
|
|
113
|
+
rfmt_parser.set_defaults(func=rfmt_cmd)
|
|
114
|
+
rfmt_parser.add_argument('-c', '--cfg', nargs=1, required=True,
|
|
115
|
+
help='Configuation file name to use.')
|
|
116
|
+
rfmt_parser.add_argument('-i', '--input', nargs=1,
|
|
117
|
+
help='Name of input file.', required=True)
|
|
118
|
+
rfmt_parser.add_argument('-o', '--output', nargs=1,
|
|
119
|
+
help='Name of output file to create.',
|
|
120
|
+
required=True)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def transform_cmd(arguments: Optional[list[str]] = None) -> None:
|
|
124
|
+
"""Command to transform list in excel or CSV file."""
|
|
125
|
+
epimain = 'More detailed help is available for each sub-command.'
|
|
126
|
+
if arguments is None: # pragma: no cover
|
|
127
|
+
arguments = sys_argv
|
|
128
|
+
fixed_args = deepcopy(arguments)
|
|
129
|
+
if len(fixed_args) > 2 and 'python' in fixed_args[0]:
|
|
130
|
+
del fixed_args[0]
|
|
131
|
+
if len(fixed_args) > 2 and '-m' == fixed_args[0]:
|
|
132
|
+
del fixed_args[0]
|
|
133
|
+
while len(fixed_args) > 1 and fixed_args[0][-3:] == '.py':
|
|
134
|
+
del fixed_args[0]
|
|
135
|
+
desc = GENERAL_DESCRIPTION + \
|
|
136
|
+
USAGE_ORDER
|
|
137
|
+
parser = argparse.ArgumentParser(description=desc, epilog=epimain)
|
|
138
|
+
subparsers = parser.add_subparsers(dest='subparser_name', required=True)
|
|
139
|
+
gen_cfg_args(subparsers)
|
|
140
|
+
rfmt_args(subparsers)
|
|
141
|
+
args = parser.parse_args(args=fixed_args)
|
|
142
|
+
_ = args.func(args)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
if __name__ == "__main__": # pragma: no cover
|
|
146
|
+
transform_cmd()
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
#! /usr/local/bin/python3
|
|
2
|
+
"""Function for doing transform of lists excel files."""
|
|
3
|
+
|
|
4
|
+
# Copyright (c) 2024 Tom Björkholm
|
|
5
|
+
# MIT License
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
from typing import Callable, Mapping
|
|
9
|
+
from excel_list_transform.config_enums import ColumnRef, FileType
|
|
10
|
+
from excel_list_transform.file_extension import fix_file_extension
|
|
11
|
+
from excel_list_transform.config_factory import config_factory_from_json
|
|
12
|
+
from excel_list_transform.config import Config
|
|
13
|
+
from excel_list_transform.transform_func_num import transform_named_files_num
|
|
14
|
+
from excel_list_transform.transform_func_named import \
|
|
15
|
+
transform_named_files_name
|
|
16
|
+
from excel_list_transform.file_must_exist import file_must_exist
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def transform_named_files(infilename: str, outfilename: str,
|
|
20
|
+
cfgfilename: str) -> int:
|
|
21
|
+
"""Transform list in the named excel file to named file."""
|
|
22
|
+
fixed_cfgfilename = fix_file_extension(filename=cfgfilename,
|
|
23
|
+
ext_to_add='.cfg')
|
|
24
|
+
file_must_exist(fixed_cfgfilename)
|
|
25
|
+
cfg = config_factory_from_json(from_json_filename=cfgfilename)
|
|
26
|
+
fixed_infilename = None
|
|
27
|
+
if cfg.in_type == FileType.CSV:
|
|
28
|
+
fixed_infilename = fix_file_extension(filename=infilename,
|
|
29
|
+
ext_to_add='.csv',
|
|
30
|
+
for_reading=True)
|
|
31
|
+
file_must_exist(fixed_infilename)
|
|
32
|
+
else:
|
|
33
|
+
fixed_infilename = fix_file_extension(filename=infilename,
|
|
34
|
+
ext_to_add='.xlsx',
|
|
35
|
+
for_reading=True)
|
|
36
|
+
file_must_exist(fixed_infilename)
|
|
37
|
+
fixed_outfilename = None
|
|
38
|
+
if cfg.out_type == FileType.CSV:
|
|
39
|
+
fixed_outfilename = fix_file_extension(filename=outfilename,
|
|
40
|
+
ext_to_add='.csv')
|
|
41
|
+
else:
|
|
42
|
+
fixed_outfilename = fix_file_extension(filename=outfilename,
|
|
43
|
+
ext_to_add='.xlsx')
|
|
44
|
+
dispatch: Mapping[ColumnRef,
|
|
45
|
+
Callable[[str, str, Config], None]] = \
|
|
46
|
+
{ColumnRef.BY_NUMBER: transform_named_files_num,
|
|
47
|
+
ColumnRef.BY_NAME: transform_named_files_name}
|
|
48
|
+
dispatch[cfg.column_ref](fixed_infilename, fixed_outfilename,
|
|
49
|
+
cfg)
|
|
50
|
+
return 0
|