python-table-converter 0.2.15__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/PKG-INFO +16 -1
- python_table_converter-0.3.0/README.md +18 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/pyproject.toml +3 -3
- python_table_converter-0.3.0/table_converter/__init__.py +16 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/commands/convert_tables.py +16 -2
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/actions.py +185 -7
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/constants.py +2 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/convert.py +82 -11
- python_table_converter-0.3.0/table_converter/core/functions/get_nested_field_value.py +22 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_nested_field_value.py +7 -2
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/types.py +19 -1
- python_table_converter-0.2.15/README.md +0 -3
- python_table_converter-0.2.15/table_converter/__init__.py +0 -2
- python_table_converter-0.2.15/table_converter/core/functions/get_nested_field_value.py +0 -15
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/LICENSE +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/cli.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/config.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/assign_id.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/flatten_row.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/nest_row.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/search_column_value.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_flat_field_value.py +0 -0
- {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_row_value.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: python-table-converter
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: A table data converter
|
|
5
5
|
Home-page: https://github.com/akivajp/python-table-converter
|
|
6
6
|
License: MIT
|
|
@@ -22,6 +22,21 @@ Requires-Dist: xlsxwriter (>=3.2.0,<4.0.0)
|
|
|
22
22
|
Project-URL: Repository, https://github.com/akivajp/python-table-converter
|
|
23
23
|
Description-Content-Type: text/markdown
|
|
24
24
|
|
|
25
|
+
# ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
|
|
26
|
+
|
|
27
|
+
This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
|
|
28
|
+
The Python package is now distributed as `tabpro` on PyPI.
|
|
29
|
+
|
|
30
|
+
```sh
|
|
31
|
+
pip install tabpro
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Migration Repository
|
|
35
|
+
- Repository: [TabPro](https://github.com/akivajp/tabpro)
|
|
36
|
+
- PyPI package: [tabpro](https://pypi.org/project/tabpro/)
|
|
37
|
+
|
|
38
|
+
---
|
|
39
|
+
|
|
25
40
|
# Table Data Converter
|
|
26
41
|
|
|
27
42
|
This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
# ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
|
|
2
|
+
|
|
3
|
+
This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
|
|
4
|
+
The Python package is now distributed as `tabpro` on PyPI.
|
|
5
|
+
|
|
6
|
+
```sh
|
|
7
|
+
pip install tabpro
|
|
8
|
+
```
|
|
9
|
+
|
|
10
|
+
## Migration Repository
|
|
11
|
+
- Repository: [TabPro](https://github.com/akivajp/tabpro)
|
|
12
|
+
- PyPI package: [tabpro](https://pypi.org/project/tabpro/)
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
# Table Data Converter
|
|
17
|
+
|
|
18
|
+
This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "python-table-converter"
|
|
3
|
-
version = "0.
|
|
3
|
+
version = "0.3.0"
|
|
4
4
|
description = "A table data converter"
|
|
5
5
|
authors = ["Akiva Miura <akiva.miura@gmail.com>"]
|
|
6
6
|
license = "MIT"
|
|
@@ -11,12 +11,12 @@ packages = [
|
|
|
11
11
|
]
|
|
12
12
|
|
|
13
13
|
[tool.poetry-dynamic-versioning]
|
|
14
|
-
enable =
|
|
14
|
+
enable = true
|
|
15
15
|
style = "pep440"
|
|
16
16
|
|
|
17
17
|
[tool.poetry.scripts]
|
|
18
18
|
table-converter = "table_converter.cli:main"
|
|
19
|
-
|
|
19
|
+
convert-tables = "table_converter.cli:command_convert_tables"
|
|
20
20
|
|
|
21
21
|
[tool.poetry.dependencies]
|
|
22
22
|
python = "^3.10"
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# 旧パッケージは TabPro (https://github.com/akivajp/tabpro) に移行済み。
|
|
2
|
+
# import・CLI 実行時に移行を促す警告を表示する。
|
|
3
|
+
# ※ DeprecationWarning は Python のデフォルトフィルタで非表示になるため、
|
|
4
|
+
# デフォルトで表示される FutureWarning を使用している。
|
|
5
|
+
import warnings
|
|
6
|
+
|
|
7
|
+
warnings.warn(
|
|
8
|
+
"The 'python-table-converter' package has been renamed to 'tabpro'. "
|
|
9
|
+
"Please install the new package instead: pip install tabpro "
|
|
10
|
+
"(https://github.com/akivajp/tabpro)",
|
|
11
|
+
FutureWarning,
|
|
12
|
+
stacklevel=2,
|
|
13
|
+
)
|
|
14
|
+
|
|
15
|
+
__version__ = "0.3.0"
|
|
16
|
+
__version_tuple__ = (0, 3, 0)
|
|
@@ -16,8 +16,10 @@ def run(
|
|
|
16
16
|
config_path = args.config,
|
|
17
17
|
list_actions = args.do_actions,
|
|
18
18
|
list_pick_columns = args.pick_columns,
|
|
19
|
+
action_delimiter = args.action_delimiter,
|
|
19
20
|
output_debug = args.output_debug,
|
|
20
21
|
verbose = args.verbose,
|
|
22
|
+
ignore_file_rows = args.ignore_file_rows,
|
|
21
23
|
)
|
|
22
24
|
|
|
23
25
|
def setup_parser(
|
|
@@ -30,9 +32,9 @@ def setup_parser(
|
|
|
30
32
|
help='Path to the input file.'
|
|
31
33
|
)
|
|
32
34
|
parser.add_argument(
|
|
33
|
-
'--output-file', '-o',
|
|
35
|
+
'--output-file', '--output', '-o',
|
|
34
36
|
metavar='OUTPUT_FILE',
|
|
35
|
-
required=
|
|
37
|
+
required=False,
|
|
36
38
|
help='Path to the output file.'
|
|
37
39
|
)
|
|
38
40
|
parser.add_argument(
|
|
@@ -52,12 +54,24 @@ def setup_parser(
|
|
|
52
54
|
nargs='+',
|
|
53
55
|
help='Pick column map',
|
|
54
56
|
)
|
|
57
|
+
parser.add_argument(
|
|
58
|
+
'--action-delimiter', '--do-delimiter', '--do-delim',
|
|
59
|
+
type=str,
|
|
60
|
+
default=':',
|
|
61
|
+
help='Action delimiter',
|
|
62
|
+
)
|
|
55
63
|
parser.add_argument(
|
|
56
64
|
'--do-actions', '--actions', '--do',
|
|
57
65
|
nargs='+',
|
|
58
66
|
type=str,
|
|
59
67
|
help='Actions to do',
|
|
60
68
|
)
|
|
69
|
+
parser.add_argument(
|
|
70
|
+
'--ignore-file-rows', '--ignore-rows', '--ignore',
|
|
71
|
+
nargs='+',
|
|
72
|
+
type=str,
|
|
73
|
+
help='Ignore tuples of file name and row index',
|
|
74
|
+
)
|
|
61
75
|
parser.add_argument(
|
|
62
76
|
'--output-debug',
|
|
63
77
|
action='store_true',
|
{python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/actions.py
RENAMED
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
Actions are used to transform the data in the table.
|
|
3
3
|
'''
|
|
4
4
|
|
|
5
|
+
import ast
|
|
6
|
+
import json
|
|
7
|
+
import re
|
|
8
|
+
|
|
5
9
|
from collections import OrderedDict
|
|
6
10
|
from dataclasses import dataclass
|
|
7
11
|
from typing import (
|
|
@@ -20,6 +24,7 @@ from . constants import (
|
|
|
20
24
|
)
|
|
21
25
|
|
|
22
26
|
from . types import (
|
|
27
|
+
AssignConfig,
|
|
23
28
|
AssignConstantConfig,
|
|
24
29
|
AssignFormatConfig,
|
|
25
30
|
AssignIdConfig,
|
|
@@ -27,6 +32,7 @@ from . types import (
|
|
|
27
32
|
GlobalStatus,
|
|
28
33
|
JoinConfig,
|
|
29
34
|
OmitConfig,
|
|
35
|
+
ParseConfig,
|
|
30
36
|
PickConfig,
|
|
31
37
|
SplitConfig,
|
|
32
38
|
Row,
|
|
@@ -45,21 +51,23 @@ from . functions.set_nested_field_value import set_nested_field_value
|
|
|
45
51
|
def setup_actions_with_args(
|
|
46
52
|
config: Config,
|
|
47
53
|
list_actions: list[str],
|
|
54
|
+
action_delimiter: str = ':',
|
|
48
55
|
):
|
|
49
56
|
ic(list_actions)
|
|
50
57
|
for str_action in list_actions:
|
|
51
|
-
fields = str_action.split(
|
|
58
|
+
fields = str_action.split(action_delimiter)
|
|
52
59
|
if len(fields) >= 1:
|
|
53
60
|
action_name = fields[0].strip()
|
|
54
61
|
if action_name == 'assign-format':
|
|
55
|
-
setup_assign_format_action(config, str_action)
|
|
62
|
+
setup_assign_format_action(config, str_action, action_delimiter)
|
|
56
63
|
continue
|
|
57
64
|
if action_name == 'filter':
|
|
58
|
-
setup_filter_action(config, str_action)
|
|
65
|
+
setup_filter_action(config, str_action, action_delimiter)
|
|
59
66
|
continue
|
|
60
67
|
if len(fields) not in [2,3]:
|
|
61
68
|
raise ValueError(
|
|
62
|
-
|
|
69
|
+
'Action must have 2 or 3 delimiter-separated fields: ' +
|
|
70
|
+
f'delimiter:{action_delimiter!r}, action string: {str_action!r}'
|
|
63
71
|
)
|
|
64
72
|
str_fields = fields[1].strip()
|
|
65
73
|
if len(fields) == 3:
|
|
@@ -83,6 +91,23 @@ def setup_actions_with_args(
|
|
|
83
91
|
else:
|
|
84
92
|
target = field.strip()
|
|
85
93
|
source = field.strip()
|
|
94
|
+
if action_name == 'assign':
|
|
95
|
+
assign_default = False
|
|
96
|
+
default_value = None
|
|
97
|
+
if 'default' in options:
|
|
98
|
+
assign_default = True
|
|
99
|
+
default_value = options['default']
|
|
100
|
+
if default_value in ['None', 'none', 'Null', 'null']:
|
|
101
|
+
default_value = None
|
|
102
|
+
required = options.get('required', False)
|
|
103
|
+
config.actions.append(AssignConfig(
|
|
104
|
+
target = target,
|
|
105
|
+
source = source,
|
|
106
|
+
assign_default = assign_default,
|
|
107
|
+
default_value = default_value,
|
|
108
|
+
required = required,
|
|
109
|
+
))
|
|
110
|
+
continue
|
|
86
111
|
if action_name == 'assign-constant':
|
|
87
112
|
str_type = options.get('type', 'str')
|
|
88
113
|
if str_type in ['str', 'string']:
|
|
@@ -112,6 +137,20 @@ def setup_actions_with_args(
|
|
|
112
137
|
context = context,
|
|
113
138
|
))
|
|
114
139
|
continue
|
|
140
|
+
if action_name == 'filter-empty':
|
|
141
|
+
config.actions.append(FilterConfig(
|
|
142
|
+
field = target,
|
|
143
|
+
operator = 'empty',
|
|
144
|
+
value = '',
|
|
145
|
+
))
|
|
146
|
+
continue
|
|
147
|
+
if action_name == 'filter-not-empty':
|
|
148
|
+
config.actions.append(FilterConfig(
|
|
149
|
+
field = target,
|
|
150
|
+
operator = 'not-empty',
|
|
151
|
+
value = '',
|
|
152
|
+
))
|
|
153
|
+
continue
|
|
115
154
|
if action_name == 'join':
|
|
116
155
|
delimiter = options.get('delimiter', None)
|
|
117
156
|
config.actions.append(JoinConfig(
|
|
@@ -125,8 +164,33 @@ def setup_actions_with_args(
|
|
|
125
164
|
field = target,
|
|
126
165
|
))
|
|
127
166
|
continue
|
|
167
|
+
if action_name == 'parse':
|
|
168
|
+
as_type = options.get('as', 'literal')
|
|
169
|
+
required = options.get('required', False)
|
|
170
|
+
if as_type not in ['json', 'literal']:
|
|
171
|
+
raise ValueError(
|
|
172
|
+
f'Unsupported as type: {as_type}'
|
|
173
|
+
)
|
|
174
|
+
config.actions.append(ParseConfig(
|
|
175
|
+
target = target,
|
|
176
|
+
source = source,
|
|
177
|
+
as_type = as_type,
|
|
178
|
+
required = required,
|
|
179
|
+
))
|
|
180
|
+
continue
|
|
181
|
+
if action_name == 'parse-json':
|
|
182
|
+
required = options.get('required', False)
|
|
183
|
+
config.actions.append(ParseConfig(
|
|
184
|
+
target = target,
|
|
185
|
+
source = source,
|
|
186
|
+
as_type = 'json',
|
|
187
|
+
required = required,
|
|
188
|
+
))
|
|
189
|
+
continue
|
|
128
190
|
if action_name == 'split':
|
|
129
191
|
delimiter = options.get('delimiter', None)
|
|
192
|
+
if delimiter == '\\n':
|
|
193
|
+
delimiter = '\n'
|
|
130
194
|
config.actions.append(SplitConfig(
|
|
131
195
|
target = target,
|
|
132
196
|
source = source,
|
|
@@ -141,8 +205,9 @@ def setup_actions_with_args(
|
|
|
141
205
|
def setup_assign_format_action(
|
|
142
206
|
config: Config,
|
|
143
207
|
str_action: str,
|
|
208
|
+
delimiter: str = ':',
|
|
144
209
|
):
|
|
145
|
-
action_fields = str_action.split(
|
|
210
|
+
action_fields = str_action.split(delimiter, 1)
|
|
146
211
|
if len(action_fields) != 2:
|
|
147
212
|
raise ValueError(
|
|
148
213
|
f'Expected 2 fields separated by ":": {str_action}'
|
|
@@ -165,8 +230,9 @@ def setup_assign_format_action(
|
|
|
165
230
|
def setup_filter_action(
|
|
166
231
|
config: Config,
|
|
167
232
|
str_action: str,
|
|
233
|
+
delimiter: str = ':',
|
|
168
234
|
):
|
|
169
|
-
action_fields = str_action.split(
|
|
235
|
+
action_fields = str_action.split(delimiter, 1)
|
|
170
236
|
if len(action_fields) != 2:
|
|
171
237
|
raise ValueError(
|
|
172
238
|
f'Expected 2 fields separated by ":": {str_action}'
|
|
@@ -190,6 +256,14 @@ def setup_filter_action(
|
|
|
190
256
|
value = value.strip(),
|
|
191
257
|
))
|
|
192
258
|
return config
|
|
259
|
+
if '=~' in str_filter:
|
|
260
|
+
field, value = str_filter.split('=~')
|
|
261
|
+
config.actions.append(FilterConfig(
|
|
262
|
+
field = field.strip(),
|
|
263
|
+
operator = '=~',
|
|
264
|
+
value = value.strip(),
|
|
265
|
+
))
|
|
266
|
+
return config
|
|
193
267
|
raise ValueError(
|
|
194
268
|
f'Unsupported filter: {str_filter}'
|
|
195
269
|
)
|
|
@@ -210,6 +284,8 @@ def do_action(
|
|
|
210
284
|
row: Row,
|
|
211
285
|
action: AssignConstantConfig,
|
|
212
286
|
):
|
|
287
|
+
if isinstance(action, AssignConfig):
|
|
288
|
+
return assign(row, action)
|
|
213
289
|
if isinstance(action, AssignConstantConfig):
|
|
214
290
|
return assign_constant(row, action)
|
|
215
291
|
if isinstance(action, AssignFormatConfig):
|
|
@@ -222,6 +298,8 @@ def do_action(
|
|
|
222
298
|
return None
|
|
223
299
|
if isinstance(action, JoinConfig):
|
|
224
300
|
return join_field(row, action)
|
|
301
|
+
if isinstance(action, ParseConfig):
|
|
302
|
+
return parse(row, action)
|
|
225
303
|
if isinstance(action, OmitConfig):
|
|
226
304
|
return omit_field(row, action)
|
|
227
305
|
if isinstance(action, SplitConfig):
|
|
@@ -235,7 +313,11 @@ def prepare_row(
|
|
|
235
313
|
):
|
|
236
314
|
if flat_row is None:
|
|
237
315
|
flat_row = OrderedDict()
|
|
238
|
-
|
|
316
|
+
try:
|
|
317
|
+
nested_row = nest_row(flat_row)
|
|
318
|
+
except:
|
|
319
|
+
ic(flat_row)
|
|
320
|
+
raise
|
|
239
321
|
return Row(
|
|
240
322
|
flat = OrderedDict(flat_row),
|
|
241
323
|
nested = nested_row,
|
|
@@ -291,6 +373,7 @@ def split_field(
|
|
|
291
373
|
if found:
|
|
292
374
|
if isinstance(value, str):
|
|
293
375
|
new_value = value.split(config.delimiter)
|
|
376
|
+
new_value = map(str.strip, new_value)
|
|
294
377
|
new_value = list(filter(None, new_value))
|
|
295
378
|
value = new_value
|
|
296
379
|
set_row_staging_value(row, config.target, value)
|
|
@@ -337,6 +420,47 @@ def remap_columns(
|
|
|
337
420
|
row.nested = nest_row(new_flat_row)
|
|
338
421
|
return row
|
|
339
422
|
|
|
423
|
+
|
|
424
|
+
def search_with_operator(
|
|
425
|
+
row: Row,
|
|
426
|
+
source: str,
|
|
427
|
+
):
|
|
428
|
+
or_operator = '\|\|'
|
|
429
|
+
null_or_operator = '\?\?'
|
|
430
|
+
operator_group = f'{or_operator}|{null_or_operator}'
|
|
431
|
+
matched = re.split(f'({operator_group})', source, 1)
|
|
432
|
+
#ic(source, matched)
|
|
433
|
+
if len(matched) == 1:
|
|
434
|
+
return search_column_value(row.nested, source)
|
|
435
|
+
matched = map(str.strip, matched)
|
|
436
|
+
left, operator, rest = matched
|
|
437
|
+
value, found = search_column_value(row.nested, left)
|
|
438
|
+
if operator == '||':
|
|
439
|
+
if bool(value):
|
|
440
|
+
return value, found
|
|
441
|
+
if operator == '??':
|
|
442
|
+
if found and value is not None:
|
|
443
|
+
return value, found
|
|
444
|
+
return search_with_operator(row, rest)
|
|
445
|
+
|
|
446
|
+
def assign(
|
|
447
|
+
row: Row,
|
|
448
|
+
config: AssignConfig,
|
|
449
|
+
):
|
|
450
|
+
value, found = search_with_operator(row, config.source)
|
|
451
|
+
if config.required:
|
|
452
|
+
if not found or bool(value) == False:
|
|
453
|
+
raise ValueError(
|
|
454
|
+
'Required field not found or empty, ' +
|
|
455
|
+
f'field: {config.source}, found: {found}, value: {value}'
|
|
456
|
+
)
|
|
457
|
+
if found:
|
|
458
|
+
set_row_staging_value(row, config.target, value)
|
|
459
|
+
else:
|
|
460
|
+
if config.assign_default:
|
|
461
|
+
set_row_staging_value(row, config.target, config.default_value)
|
|
462
|
+
return row
|
|
463
|
+
|
|
340
464
|
def assign_format(
|
|
341
465
|
row: Row,
|
|
342
466
|
config: AssignFormatConfig,
|
|
@@ -369,6 +493,14 @@ def assign_format(
|
|
|
369
493
|
set_row_staging_value(row, config.target, formatted)
|
|
370
494
|
return row
|
|
371
495
|
|
|
496
|
+
def check_empty(
|
|
497
|
+
value: Any,
|
|
498
|
+
found: str | None,
|
|
499
|
+
):
|
|
500
|
+
if not found:
|
|
501
|
+
return True
|
|
502
|
+
return not bool(value)
|
|
503
|
+
|
|
372
504
|
def filter_row(
|
|
373
505
|
row: Row,
|
|
374
506
|
config: list[FilterConfig],
|
|
@@ -383,6 +515,11 @@ def filter_row(
|
|
|
383
515
|
elif config.operator == '!=':
|
|
384
516
|
if str(value) == str(config.value) or value == config.value:
|
|
385
517
|
return False
|
|
518
|
+
elif config.operator == '=~':
|
|
519
|
+
if not found:
|
|
520
|
+
return False
|
|
521
|
+
if not re.search(config.value, value):
|
|
522
|
+
return False
|
|
386
523
|
elif config.operator == 'not-in':
|
|
387
524
|
if isinstance(config.value, list):
|
|
388
525
|
if value in config.value:
|
|
@@ -391,6 +528,12 @@ def filter_row(
|
|
|
391
528
|
return False
|
|
392
529
|
else:
|
|
393
530
|
raise ValueError(f'Unsupported filter value type: type{config.value}')
|
|
531
|
+
elif config.operator == 'empty':
|
|
532
|
+
if not check_empty(value, found):
|
|
533
|
+
return False
|
|
534
|
+
elif config.operator == 'not-empty':
|
|
535
|
+
if check_empty(value, found):
|
|
536
|
+
return False
|
|
394
537
|
else:
|
|
395
538
|
raise ValueError(f'Unsupported operator: {config.operator}')
|
|
396
539
|
return True
|
|
@@ -421,3 +564,38 @@ def join_field(
|
|
|
421
564
|
value = delimiter.join(value)
|
|
422
565
|
set_row_staging_value(row, config.target, value)
|
|
423
566
|
return row
|
|
567
|
+
|
|
568
|
+
def parse(
|
|
569
|
+
row: Row,
|
|
570
|
+
config: AssignConfig,
|
|
571
|
+
):
|
|
572
|
+
value, found = search_column_value(row.nested, config.source)
|
|
573
|
+
if config.required:
|
|
574
|
+
if not found:
|
|
575
|
+
raise ValueError(
|
|
576
|
+
f'Required field not found, field: {config.source}'
|
|
577
|
+
)
|
|
578
|
+
if found:
|
|
579
|
+
if type(value) == str:
|
|
580
|
+
if config.as_type == 'literal':
|
|
581
|
+
try:
|
|
582
|
+
parsed = ast.literal_eval(value)
|
|
583
|
+
except:
|
|
584
|
+
raise ValueError(
|
|
585
|
+
f'Failed to parse literal: {value}'
|
|
586
|
+
)
|
|
587
|
+
elif config.as_type == 'json':
|
|
588
|
+
try:
|
|
589
|
+
parsed = json.loads(value)
|
|
590
|
+
except:
|
|
591
|
+
raise ValueError(
|
|
592
|
+
f'Failed to parse JSON: {value}'
|
|
593
|
+
)
|
|
594
|
+
else:
|
|
595
|
+
raise ValueError(
|
|
596
|
+
f'Unsupported as type: {config.as_type}'
|
|
597
|
+
)
|
|
598
|
+
else:
|
|
599
|
+
parsed = value
|
|
600
|
+
set_row_staging_value(row, config.target, parsed)
|
|
601
|
+
return row
|
{python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/convert.py
RENAMED
|
@@ -26,6 +26,8 @@ from . config import (
|
|
|
26
26
|
)
|
|
27
27
|
from . constants import (
|
|
28
28
|
FILE_FIELD,
|
|
29
|
+
ROW_INDEX_FIELD,
|
|
30
|
+
FILE_ROW_INDEX_FIELD,
|
|
29
31
|
INPUT_FIELD,
|
|
30
32
|
STAGING_FIELD,
|
|
31
33
|
)
|
|
@@ -70,11 +72,34 @@ def register_saver(
|
|
|
70
72
|
return saver
|
|
71
73
|
return decorator
|
|
72
74
|
|
|
75
|
+
@register_loader('.csv')
|
|
76
|
+
def load_csv(
|
|
77
|
+
input_file: str,
|
|
78
|
+
):
|
|
79
|
+
# utf-8
|
|
80
|
+
#df = pd.read_csv(input_file)
|
|
81
|
+
# UTF-8 with BOM
|
|
82
|
+
df = pd.read_csv(input_file, encoding='utf-8-sig')
|
|
83
|
+
return df
|
|
84
|
+
|
|
73
85
|
@register_loader('.xlsx')
|
|
74
86
|
def load_excel(
|
|
75
87
|
input_file: str,
|
|
76
88
|
):
|
|
77
|
-
df = pd.read_excel(input_file)
|
|
89
|
+
#df = pd.read_excel(input_file)
|
|
90
|
+
# NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
|
|
91
|
+
df = pd.read_excel(input_file, dtype=str)
|
|
92
|
+
# NOTE: 列番号でもアクセスできるようフィールドを追加する
|
|
93
|
+
df_with_column_number = pd.read_excel(
|
|
94
|
+
input_file, dtype=str, header=None, skiprows=1
|
|
95
|
+
)
|
|
96
|
+
new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
|
|
97
|
+
df2 = df_with_column_number.rename(columns=dict(
|
|
98
|
+
zip(df_with_column_number.columns, new_column_names)
|
|
99
|
+
))
|
|
100
|
+
df = pd.concat([df, df2], axis=1)
|
|
101
|
+
df = df.dropna(axis=0, how='all')
|
|
102
|
+
df = df.dropna(axis=1, how='all')
|
|
78
103
|
return df
|
|
79
104
|
|
|
80
105
|
@register_loader('.json')
|
|
@@ -119,6 +144,18 @@ def save_json(
|
|
|
119
144
|
ensure_ascii=False,
|
|
120
145
|
)
|
|
121
146
|
|
|
147
|
+
@register_loader('.jsonl')
|
|
148
|
+
def load_jsonl(
|
|
149
|
+
input_file: str,
|
|
150
|
+
):
|
|
151
|
+
rows = []
|
|
152
|
+
with open(input_file, 'r') as f:
|
|
153
|
+
for line in f:
|
|
154
|
+
row = json.loads(line)
|
|
155
|
+
rows.append(row)
|
|
156
|
+
df = pd.DataFrame(rows)
|
|
157
|
+
return df
|
|
158
|
+
|
|
122
159
|
@register_saver('.jsonl')
|
|
123
160
|
def save_jsonl(
|
|
124
161
|
df: pd.DataFrame,
|
|
@@ -249,20 +286,29 @@ def convert(
|
|
|
249
286
|
output_debug: bool = False,
|
|
250
287
|
list_actions: list[str] | None = None,
|
|
251
288
|
list_pick_columns: list[str] | None = None,
|
|
289
|
+
action_delimiter: str = ':',
|
|
252
290
|
verbose: bool = False,
|
|
291
|
+
ignore_file_rows: list[str] | None = None,
|
|
253
292
|
):
|
|
254
293
|
ic.enable()
|
|
255
294
|
ic()
|
|
256
295
|
ic(input_files)
|
|
257
296
|
df_list = []
|
|
258
297
|
row_list_filtered_out = []
|
|
298
|
+
set_ignore_file_rows = set()
|
|
259
299
|
global_status = GlobalStatus()
|
|
260
300
|
config = setup_config(config_path)
|
|
261
301
|
ic(config)
|
|
302
|
+
if ignore_file_rows:
|
|
303
|
+
set_ignore_file_rows = set(ignore_file_rows)
|
|
262
304
|
if list_pick_columns:
|
|
263
305
|
setup_pick_with_args(config, list_pick_columns)
|
|
264
306
|
if list_actions:
|
|
265
|
-
setup_actions_with_args(
|
|
307
|
+
setup_actions_with_args(
|
|
308
|
+
config,
|
|
309
|
+
list_actions,
|
|
310
|
+
action_delimiter=action_delimiter
|
|
311
|
+
)
|
|
266
312
|
if output_file:
|
|
267
313
|
ext = os.path.splitext(output_file)[1]
|
|
268
314
|
if ext not in dict_savers:
|
|
@@ -274,6 +320,7 @@ def convert(
|
|
|
274
320
|
ic(input_file)
|
|
275
321
|
if not os.path.exists(input_file):
|
|
276
322
|
raise FileNotFoundError(f'File not found: {input_file}')
|
|
323
|
+
base_name = os.path.basename(input_file)
|
|
277
324
|
ext = os.path.splitext(input_file)[1]
|
|
278
325
|
ic(ext)
|
|
279
326
|
if ext not in dict_loaders:
|
|
@@ -288,10 +335,20 @@ def convert(
|
|
|
288
335
|
#new_rows = []
|
|
289
336
|
new_flat_rows = []
|
|
290
337
|
for index, flat_row in df.iterrows():
|
|
338
|
+
file_row_index = f'{input_file}:{index}'
|
|
339
|
+
if file_row_index in set_ignore_file_rows:
|
|
340
|
+
continue
|
|
341
|
+
short_file_row_index = f'{base_name}:{index}'
|
|
342
|
+
if short_file_row_index in set_ignore_file_rows:
|
|
343
|
+
continue
|
|
344
|
+
#if flat_row.empty:
|
|
345
|
+
# continue
|
|
291
346
|
orig_row = prepare_row(flat_row)
|
|
292
347
|
row = prepare_row(flat_row)
|
|
293
348
|
if STAGING_FIELD not in orig_row.nested:
|
|
294
349
|
set_row_staging_value(row, FILE_FIELD, input_file)
|
|
350
|
+
set_row_staging_value(row, FILE_ROW_INDEX_FIELD, file_row_index)
|
|
351
|
+
set_row_staging_value(row, ROW_INDEX_FIELD, index)
|
|
295
352
|
set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
|
|
296
353
|
if config.process.assign_array:
|
|
297
354
|
row.flat= assign_array(row.flat, config.process.assign_array)
|
|
@@ -300,16 +357,23 @@ def convert(
|
|
|
300
357
|
if config.process.assign_length:
|
|
301
358
|
row.flat = assign_length(row.flat, config.process.assign_length)
|
|
302
359
|
if config.actions:
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
if
|
|
306
|
-
|
|
360
|
+
try:
|
|
361
|
+
new_row = do_actions(global_status, row, config.actions)
|
|
362
|
+
if new_row is None:
|
|
363
|
+
if not output_debug:
|
|
364
|
+
pop_row_staging(row)
|
|
365
|
+
if verbose:
|
|
366
|
+
ic('Filtered out: ', row.flat)
|
|
367
|
+
if output_file_filtered_out:
|
|
368
|
+
row_list_filtered_out.append(row.flat)
|
|
369
|
+
continue
|
|
370
|
+
row = new_row
|
|
371
|
+
except Exception as e:
|
|
307
372
|
if verbose:
|
|
308
|
-
ic(
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
row = new_row
|
|
373
|
+
ic(index)
|
|
374
|
+
ic(flat_row)
|
|
375
|
+
ic(row.flat)
|
|
376
|
+
raise e
|
|
313
377
|
if config.pick:
|
|
314
378
|
remap_columns(row, config.pick)
|
|
315
379
|
if not output_debug:
|
|
@@ -317,6 +381,11 @@ def convert(
|
|
|
317
381
|
new_flat_rows.append(row.flat)
|
|
318
382
|
new_df = pd.DataFrame(new_flat_rows)
|
|
319
383
|
df_list.append(new_df)
|
|
384
|
+
# NOTE: concatの仕様が変わり、all-NAの列を含むdfを連結しようとすると警告が出るようになった
|
|
385
|
+
#if ic(new_df.dropna(axis=1, how='all').empty):
|
|
386
|
+
# ic(new_df.dropna(axis=1, how='all'))
|
|
387
|
+
# raise ValueError('No rows to output.')
|
|
388
|
+
#df_list.append(new_df.dropna(axis=1, how='all'))
|
|
320
389
|
all_df = pd.concat(df_list)
|
|
321
390
|
#ic(all_df)
|
|
322
391
|
ic(len(all_df))
|
|
@@ -325,6 +394,8 @@ def convert(
|
|
|
325
394
|
if output_file:
|
|
326
395
|
ic('Saing to: ', output_file)
|
|
327
396
|
saver(all_df, output_file)
|
|
397
|
+
else:
|
|
398
|
+
ic(all_df)
|
|
328
399
|
if row_list_filtered_out:
|
|
329
400
|
df_filtered_out = pd.DataFrame(row_list_filtered_out)
|
|
330
401
|
ic('Saving filtered out to: ', output_file_filtered_out)
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Description: Get the value of a field in a dictionary.
|
|
2
|
+
|
|
3
|
+
from collections import OrderedDict
|
|
4
|
+
|
|
5
|
+
def get_nested_field_value(
|
|
6
|
+
data: OrderedDict | list,
|
|
7
|
+
field: str,
|
|
8
|
+
):
|
|
9
|
+
if isinstance(data, list):
|
|
10
|
+
if field.isdigit():
|
|
11
|
+
index = int(field)
|
|
12
|
+
if index < len(data):
|
|
13
|
+
return data[index], True
|
|
14
|
+
return None, False
|
|
15
|
+
if isinstance(data, dict):
|
|
16
|
+
if field in data:
|
|
17
|
+
return data[field], True
|
|
18
|
+
if '.' in field:
|
|
19
|
+
field, rest = field.split('.', 1)
|
|
20
|
+
if field in data:
|
|
21
|
+
return get_nested_field_value(data[field], rest)
|
|
22
|
+
return None, False
|
|
@@ -12,8 +12,13 @@ def set_nested_field_value(
|
|
|
12
12
|
):
|
|
13
13
|
if isinstance(field, str) and '.' in field:
|
|
14
14
|
field, rest = field.split('.', 1)
|
|
15
|
-
|
|
15
|
+
sub_data = data.get(field)
|
|
16
|
+
if not isinstance(sub_data, dict):
|
|
16
17
|
data[field] = OrderedDict()
|
|
17
18
|
set_nested_field_value(data[field], rest, value)
|
|
18
19
|
else:
|
|
19
|
-
|
|
20
|
+
try:
|
|
21
|
+
data[field] = value
|
|
22
|
+
except:
|
|
23
|
+
ic(data, field, value)
|
|
24
|
+
raise
|
{python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/types.py
RENAMED
|
@@ -21,6 +21,14 @@ class Row:
|
|
|
21
21
|
flat: OrderedDict
|
|
22
22
|
nested: OrderedDict
|
|
23
23
|
|
|
24
|
+
@dataclasses.dataclass
|
|
25
|
+
class AssignConfig:
|
|
26
|
+
target: str
|
|
27
|
+
source: str
|
|
28
|
+
assign_default: bool = False
|
|
29
|
+
default_value: Any = None
|
|
30
|
+
required: bool = False
|
|
31
|
+
|
|
24
32
|
@dataclasses.dataclass
|
|
25
33
|
class AssignConstantConfig:
|
|
26
34
|
target: str
|
|
@@ -40,7 +48,10 @@ class AssignIdConfig:
|
|
|
40
48
|
@dataclasses.dataclass
|
|
41
49
|
class FilterConfig:
|
|
42
50
|
field: str
|
|
43
|
-
operator: Literal[
|
|
51
|
+
operator: Literal[
|
|
52
|
+
'==', '!=', '>', '>=', '<', '<=', '=~', 'not-in',
|
|
53
|
+
'empty', 'not-empty',
|
|
54
|
+
]
|
|
44
55
|
value: str | list[str]
|
|
45
56
|
|
|
46
57
|
@dataclasses.dataclass
|
|
@@ -53,6 +64,13 @@ class JoinConfig:
|
|
|
53
64
|
class OmitConfig:
|
|
54
65
|
field: str
|
|
55
66
|
|
|
67
|
+
@dataclasses.dataclass
|
|
68
|
+
class ParseConfig:
|
|
69
|
+
target: str
|
|
70
|
+
source: str
|
|
71
|
+
as_type: Literal['json', 'literal']
|
|
72
|
+
required: bool = False
|
|
73
|
+
|
|
56
74
|
@dataclasses.dataclass
|
|
57
75
|
class PickConfig:
|
|
58
76
|
target: str
|
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
# Description: Get the value of a field in a dictionary.
|
|
2
|
-
|
|
3
|
-
from collections import OrderedDict
|
|
4
|
-
|
|
5
|
-
def get_nested_field_value(
|
|
6
|
-
data: OrderedDict,
|
|
7
|
-
field: str,
|
|
8
|
-
):
|
|
9
|
-
if field in data:
|
|
10
|
-
return data[field], True
|
|
11
|
-
if '.' in field:
|
|
12
|
-
field, rest = field.split('.', 1)
|
|
13
|
-
if field in data:
|
|
14
|
-
return get_nested_field_value(data[field], rest)
|
|
15
|
-
return None, False
|
|
File without changes
|
|
File without changes
|
{python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/config.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|