python-table-converter 0.2.15__tar.gz → 0.3.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (23) hide show
  1. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/PKG-INFO +16 -1
  2. python_table_converter-0.3.0/README.md +18 -0
  3. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/pyproject.toml +3 -3
  4. python_table_converter-0.3.0/table_converter/__init__.py +16 -0
  5. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/commands/convert_tables.py +16 -2
  6. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/actions.py +185 -7
  7. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/constants.py +2 -0
  8. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/convert.py +82 -11
  9. python_table_converter-0.3.0/table_converter/core/functions/get_nested_field_value.py +22 -0
  10. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_nested_field_value.py +7 -2
  11. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/types.py +19 -1
  12. python_table_converter-0.2.15/README.md +0 -3
  13. python_table_converter-0.2.15/table_converter/__init__.py +0 -2
  14. python_table_converter-0.2.15/table_converter/core/functions/get_nested_field_value.py +0 -15
  15. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/LICENSE +0 -0
  16. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/cli.py +0 -0
  17. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/config.py +0 -0
  18. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/assign_id.py +0 -0
  19. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/flatten_row.py +0 -0
  20. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/nest_row.py +0 -0
  21. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/search_column_value.py +0 -0
  22. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_flat_field_value.py +0 -0
  23. {python_table_converter-0.2.15 → python_table_converter-0.3.0}/table_converter/core/functions/set_row_value.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.1
2
2
  Name: python-table-converter
3
- Version: 0.2.15
3
+ Version: 0.3.0
4
4
  Summary: A table data converter
5
5
  Home-page: https://github.com/akivajp/python-table-converter
6
6
  License: MIT
@@ -22,6 +22,21 @@ Requires-Dist: xlsxwriter (>=3.2.0,<4.0.0)
22
22
  Project-URL: Repository, https://github.com/akivajp/python-table-converter
23
23
  Description-Content-Type: text/markdown
24
24
 
25
+ # ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
26
+
27
+ This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
28
+ The Python package is now distributed as `tabpro` on PyPI.
29
+
30
+ ```sh
31
+ pip install tabpro
32
+ ```
33
+
34
+ ## Migration Repository
35
+ - Repository: [TabPro](https://github.com/akivajp/tabpro)
36
+ - PyPI package: [tabpro](https://pypi.org/project/tabpro/)
37
+
38
+ ---
39
+
25
40
  # Table Data Converter
26
41
 
27
42
  This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
@@ -0,0 +1,18 @@
1
+ # ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
2
+
3
+ This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
4
+ The Python package is now distributed as `tabpro` on PyPI.
5
+
6
+ ```sh
7
+ pip install tabpro
8
+ ```
9
+
10
+ ## Migration Repository
11
+ - Repository: [TabPro](https://github.com/akivajp/tabpro)
12
+ - PyPI package: [tabpro](https://pypi.org/project/tabpro/)
13
+
14
+ ---
15
+
16
+ # Table Data Converter
17
+
18
+ This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
@@ -1,6 +1,6 @@
1
1
  [tool.poetry]
2
2
  name = "python-table-converter"
3
- version = "0.2.15"
3
+ version = "0.3.0"
4
4
  description = "A table data converter"
5
5
  authors = ["Akiva Miura <akiva.miura@gmail.com>"]
6
6
  license = "MIT"
@@ -11,12 +11,12 @@ packages = [
11
11
  ]
12
12
 
13
13
  [tool.poetry-dynamic-versioning]
14
- enable = false
14
+ enable = true
15
15
  style = "pep440"
16
16
 
17
17
  [tool.poetry.scripts]
18
18
  table-converter = "table_converter.cli:main"
19
- conver-tables = "table_converter.cli:command_convert_tables"
19
+ convert-tables = "table_converter.cli:command_convert_tables"
20
20
 
21
21
  [tool.poetry.dependencies]
22
22
  python = "^3.10"
@@ -0,0 +1,16 @@
1
+ # 旧パッケージは TabPro (https://github.com/akivajp/tabpro) に移行済み。
2
+ # import・CLI 実行時に移行を促す警告を表示する。
3
+ # ※ DeprecationWarning は Python のデフォルトフィルタで非表示になるため、
4
+ # デフォルトで表示される FutureWarning を使用している。
5
+ import warnings
6
+
7
+ warnings.warn(
8
+ "The 'python-table-converter' package has been renamed to 'tabpro'. "
9
+ "Please install the new package instead: pip install tabpro "
10
+ "(https://github.com/akivajp/tabpro)",
11
+ FutureWarning,
12
+ stacklevel=2,
13
+ )
14
+
15
+ __version__ = "0.3.0"
16
+ __version_tuple__ = (0, 3, 0)
@@ -16,8 +16,10 @@ def run(
16
16
  config_path = args.config,
17
17
  list_actions = args.do_actions,
18
18
  list_pick_columns = args.pick_columns,
19
+ action_delimiter = args.action_delimiter,
19
20
  output_debug = args.output_debug,
20
21
  verbose = args.verbose,
22
+ ignore_file_rows = args.ignore_file_rows,
21
23
  )
22
24
 
23
25
  def setup_parser(
@@ -30,9 +32,9 @@ def setup_parser(
30
32
  help='Path to the input file.'
31
33
  )
32
34
  parser.add_argument(
33
- '--output-file', '-o',
35
+ '--output-file', '--output', '-o',
34
36
  metavar='OUTPUT_FILE',
35
- required=True,
37
+ required=False,
36
38
  help='Path to the output file.'
37
39
  )
38
40
  parser.add_argument(
@@ -52,12 +54,24 @@ def setup_parser(
52
54
  nargs='+',
53
55
  help='Pick column map',
54
56
  )
57
+ parser.add_argument(
58
+ '--action-delimiter', '--do-delimiter', '--do-delim',
59
+ type=str,
60
+ default=':',
61
+ help='Action delimiter',
62
+ )
55
63
  parser.add_argument(
56
64
  '--do-actions', '--actions', '--do',
57
65
  nargs='+',
58
66
  type=str,
59
67
  help='Actions to do',
60
68
  )
69
+ parser.add_argument(
70
+ '--ignore-file-rows', '--ignore-rows', '--ignore',
71
+ nargs='+',
72
+ type=str,
73
+ help='Ignore tuples of file name and row index',
74
+ )
61
75
  parser.add_argument(
62
76
  '--output-debug',
63
77
  action='store_true',
@@ -2,6 +2,10 @@
2
2
  Actions are used to transform the data in the table.
3
3
  '''
4
4
 
5
+ import ast
6
+ import json
7
+ import re
8
+
5
9
  from collections import OrderedDict
6
10
  from dataclasses import dataclass
7
11
  from typing import (
@@ -20,6 +24,7 @@ from . constants import (
20
24
  )
21
25
 
22
26
  from . types import (
27
+ AssignConfig,
23
28
  AssignConstantConfig,
24
29
  AssignFormatConfig,
25
30
  AssignIdConfig,
@@ -27,6 +32,7 @@ from . types import (
27
32
  GlobalStatus,
28
33
  JoinConfig,
29
34
  OmitConfig,
35
+ ParseConfig,
30
36
  PickConfig,
31
37
  SplitConfig,
32
38
  Row,
@@ -45,21 +51,23 @@ from . functions.set_nested_field_value import set_nested_field_value
45
51
  def setup_actions_with_args(
46
52
  config: Config,
47
53
  list_actions: list[str],
54
+ action_delimiter: str = ':',
48
55
  ):
49
56
  ic(list_actions)
50
57
  for str_action in list_actions:
51
- fields = str_action.split(':')
58
+ fields = str_action.split(action_delimiter)
52
59
  if len(fields) >= 1:
53
60
  action_name = fields[0].strip()
54
61
  if action_name == 'assign-format':
55
- setup_assign_format_action(config, str_action)
62
+ setup_assign_format_action(config, str_action, action_delimiter)
56
63
  continue
57
64
  if action_name == 'filter':
58
- setup_filter_action(config, str_action)
65
+ setup_filter_action(config, str_action, action_delimiter)
59
66
  continue
60
67
  if len(fields) not in [2,3]:
61
68
  raise ValueError(
62
- f'Action must have 2 or 3 colon-separated fields: {str_action}'
69
+ 'Action must have 2 or 3 delimiter-separated fields: ' +
70
+ f'delimiter:{action_delimiter!r}, action string: {str_action!r}'
63
71
  )
64
72
  str_fields = fields[1].strip()
65
73
  if len(fields) == 3:
@@ -83,6 +91,23 @@ def setup_actions_with_args(
83
91
  else:
84
92
  target = field.strip()
85
93
  source = field.strip()
94
+ if action_name == 'assign':
95
+ assign_default = False
96
+ default_value = None
97
+ if 'default' in options:
98
+ assign_default = True
99
+ default_value = options['default']
100
+ if default_value in ['None', 'none', 'Null', 'null']:
101
+ default_value = None
102
+ required = options.get('required', False)
103
+ config.actions.append(AssignConfig(
104
+ target = target,
105
+ source = source,
106
+ assign_default = assign_default,
107
+ default_value = default_value,
108
+ required = required,
109
+ ))
110
+ continue
86
111
  if action_name == 'assign-constant':
87
112
  str_type = options.get('type', 'str')
88
113
  if str_type in ['str', 'string']:
@@ -112,6 +137,20 @@ def setup_actions_with_args(
112
137
  context = context,
113
138
  ))
114
139
  continue
140
+ if action_name == 'filter-empty':
141
+ config.actions.append(FilterConfig(
142
+ field = target,
143
+ operator = 'empty',
144
+ value = '',
145
+ ))
146
+ continue
147
+ if action_name == 'filter-not-empty':
148
+ config.actions.append(FilterConfig(
149
+ field = target,
150
+ operator = 'not-empty',
151
+ value = '',
152
+ ))
153
+ continue
115
154
  if action_name == 'join':
116
155
  delimiter = options.get('delimiter', None)
117
156
  config.actions.append(JoinConfig(
@@ -125,8 +164,33 @@ def setup_actions_with_args(
125
164
  field = target,
126
165
  ))
127
166
  continue
167
+ if action_name == 'parse':
168
+ as_type = options.get('as', 'literal')
169
+ required = options.get('required', False)
170
+ if as_type not in ['json', 'literal']:
171
+ raise ValueError(
172
+ f'Unsupported as type: {as_type}'
173
+ )
174
+ config.actions.append(ParseConfig(
175
+ target = target,
176
+ source = source,
177
+ as_type = as_type,
178
+ required = required,
179
+ ))
180
+ continue
181
+ if action_name == 'parse-json':
182
+ required = options.get('required', False)
183
+ config.actions.append(ParseConfig(
184
+ target = target,
185
+ source = source,
186
+ as_type = 'json',
187
+ required = required,
188
+ ))
189
+ continue
128
190
  if action_name == 'split':
129
191
  delimiter = options.get('delimiter', None)
192
+ if delimiter == '\\n':
193
+ delimiter = '\n'
130
194
  config.actions.append(SplitConfig(
131
195
  target = target,
132
196
  source = source,
@@ -141,8 +205,9 @@ def setup_actions_with_args(
141
205
  def setup_assign_format_action(
142
206
  config: Config,
143
207
  str_action: str,
208
+ delimiter: str = ':',
144
209
  ):
145
- action_fields = str_action.split(':', 1)
210
+ action_fields = str_action.split(delimiter, 1)
146
211
  if len(action_fields) != 2:
147
212
  raise ValueError(
148
213
  f'Expected 2 fields separated by ":": {str_action}'
@@ -165,8 +230,9 @@ def setup_assign_format_action(
165
230
  def setup_filter_action(
166
231
  config: Config,
167
232
  str_action: str,
233
+ delimiter: str = ':',
168
234
  ):
169
- action_fields = str_action.split(':', 1)
235
+ action_fields = str_action.split(delimiter, 1)
170
236
  if len(action_fields) != 2:
171
237
  raise ValueError(
172
238
  f'Expected 2 fields separated by ":": {str_action}'
@@ -190,6 +256,14 @@ def setup_filter_action(
190
256
  value = value.strip(),
191
257
  ))
192
258
  return config
259
+ if '=~' in str_filter:
260
+ field, value = str_filter.split('=~')
261
+ config.actions.append(FilterConfig(
262
+ field = field.strip(),
263
+ operator = '=~',
264
+ value = value.strip(),
265
+ ))
266
+ return config
193
267
  raise ValueError(
194
268
  f'Unsupported filter: {str_filter}'
195
269
  )
@@ -210,6 +284,8 @@ def do_action(
210
284
  row: Row,
211
285
  action: AssignConstantConfig,
212
286
  ):
287
+ if isinstance(action, AssignConfig):
288
+ return assign(row, action)
213
289
  if isinstance(action, AssignConstantConfig):
214
290
  return assign_constant(row, action)
215
291
  if isinstance(action, AssignFormatConfig):
@@ -222,6 +298,8 @@ def do_action(
222
298
  return None
223
299
  if isinstance(action, JoinConfig):
224
300
  return join_field(row, action)
301
+ if isinstance(action, ParseConfig):
302
+ return parse(row, action)
225
303
  if isinstance(action, OmitConfig):
226
304
  return omit_field(row, action)
227
305
  if isinstance(action, SplitConfig):
@@ -235,7 +313,11 @@ def prepare_row(
235
313
  ):
236
314
  if flat_row is None:
237
315
  flat_row = OrderedDict()
238
- nested_row = nest_row(flat_row)
316
+ try:
317
+ nested_row = nest_row(flat_row)
318
+ except:
319
+ ic(flat_row)
320
+ raise
239
321
  return Row(
240
322
  flat = OrderedDict(flat_row),
241
323
  nested = nested_row,
@@ -291,6 +373,7 @@ def split_field(
291
373
  if found:
292
374
  if isinstance(value, str):
293
375
  new_value = value.split(config.delimiter)
376
+ new_value = map(str.strip, new_value)
294
377
  new_value = list(filter(None, new_value))
295
378
  value = new_value
296
379
  set_row_staging_value(row, config.target, value)
@@ -337,6 +420,47 @@ def remap_columns(
337
420
  row.nested = nest_row(new_flat_row)
338
421
  return row
339
422
 
423
+
424
+ def search_with_operator(
425
+ row: Row,
426
+ source: str,
427
+ ):
428
+ or_operator = '\|\|'
429
+ null_or_operator = '\?\?'
430
+ operator_group = f'{or_operator}|{null_or_operator}'
431
+ matched = re.split(f'({operator_group})', source, 1)
432
+ #ic(source, matched)
433
+ if len(matched) == 1:
434
+ return search_column_value(row.nested, source)
435
+ matched = map(str.strip, matched)
436
+ left, operator, rest = matched
437
+ value, found = search_column_value(row.nested, left)
438
+ if operator == '||':
439
+ if bool(value):
440
+ return value, found
441
+ if operator == '??':
442
+ if found and value is not None:
443
+ return value, found
444
+ return search_with_operator(row, rest)
445
+
446
+ def assign(
447
+ row: Row,
448
+ config: AssignConfig,
449
+ ):
450
+ value, found = search_with_operator(row, config.source)
451
+ if config.required:
452
+ if not found or bool(value) == False:
453
+ raise ValueError(
454
+ 'Required field not found or empty, ' +
455
+ f'field: {config.source}, found: {found}, value: {value}'
456
+ )
457
+ if found:
458
+ set_row_staging_value(row, config.target, value)
459
+ else:
460
+ if config.assign_default:
461
+ set_row_staging_value(row, config.target, config.default_value)
462
+ return row
463
+
340
464
  def assign_format(
341
465
  row: Row,
342
466
  config: AssignFormatConfig,
@@ -369,6 +493,14 @@ def assign_format(
369
493
  set_row_staging_value(row, config.target, formatted)
370
494
  return row
371
495
 
496
+ def check_empty(
497
+ value: Any,
498
+ found: str | None,
499
+ ):
500
+ if not found:
501
+ return True
502
+ return not bool(value)
503
+
372
504
  def filter_row(
373
505
  row: Row,
374
506
  config: list[FilterConfig],
@@ -383,6 +515,11 @@ def filter_row(
383
515
  elif config.operator == '!=':
384
516
  if str(value) == str(config.value) or value == config.value:
385
517
  return False
518
+ elif config.operator == '=~':
519
+ if not found:
520
+ return False
521
+ if not re.search(config.value, value):
522
+ return False
386
523
  elif config.operator == 'not-in':
387
524
  if isinstance(config.value, list):
388
525
  if value in config.value:
@@ -391,6 +528,12 @@ def filter_row(
391
528
  return False
392
529
  else:
393
530
  raise ValueError(f'Unsupported filter value type: type{config.value}')
531
+ elif config.operator == 'empty':
532
+ if not check_empty(value, found):
533
+ return False
534
+ elif config.operator == 'not-empty':
535
+ if check_empty(value, found):
536
+ return False
394
537
  else:
395
538
  raise ValueError(f'Unsupported operator: {config.operator}')
396
539
  return True
@@ -421,3 +564,38 @@ def join_field(
421
564
  value = delimiter.join(value)
422
565
  set_row_staging_value(row, config.target, value)
423
566
  return row
567
+
568
+ def parse(
569
+ row: Row,
570
+ config: AssignConfig,
571
+ ):
572
+ value, found = search_column_value(row.nested, config.source)
573
+ if config.required:
574
+ if not found:
575
+ raise ValueError(
576
+ f'Required field not found, field: {config.source}'
577
+ )
578
+ if found:
579
+ if type(value) == str:
580
+ if config.as_type == 'literal':
581
+ try:
582
+ parsed = ast.literal_eval(value)
583
+ except:
584
+ raise ValueError(
585
+ f'Failed to parse literal: {value}'
586
+ )
587
+ elif config.as_type == 'json':
588
+ try:
589
+ parsed = json.loads(value)
590
+ except:
591
+ raise ValueError(
592
+ f'Failed to parse JSON: {value}'
593
+ )
594
+ else:
595
+ raise ValueError(
596
+ f'Unsupported as type: {config.as_type}'
597
+ )
598
+ else:
599
+ parsed = value
600
+ set_row_staging_value(row, config.target, parsed)
601
+ return row
@@ -1,3 +1,5 @@
1
1
  FILE_FIELD = '__file__'
2
+ ROW_INDEX_FIELD = '__row_index__'
3
+ FILE_ROW_INDEX_FIELD = '__file_row_index__'
2
4
  INPUT_FIELD = '__input__'
3
5
  STAGING_FIELD = '__staging__'
@@ -26,6 +26,8 @@ from . config import (
26
26
  )
27
27
  from . constants import (
28
28
  FILE_FIELD,
29
+ ROW_INDEX_FIELD,
30
+ FILE_ROW_INDEX_FIELD,
29
31
  INPUT_FIELD,
30
32
  STAGING_FIELD,
31
33
  )
@@ -70,11 +72,34 @@ def register_saver(
70
72
  return saver
71
73
  return decorator
72
74
 
75
+ @register_loader('.csv')
76
+ def load_csv(
77
+ input_file: str,
78
+ ):
79
+ # utf-8
80
+ #df = pd.read_csv(input_file)
81
+ # UTF-8 with BOM
82
+ df = pd.read_csv(input_file, encoding='utf-8-sig')
83
+ return df
84
+
73
85
  @register_loader('.xlsx')
74
86
  def load_excel(
75
87
  input_file: str,
76
88
  ):
77
- df = pd.read_excel(input_file)
89
+ #df = pd.read_excel(input_file)
90
+ # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
91
+ df = pd.read_excel(input_file, dtype=str)
92
+ # NOTE: 列番号でもアクセスできるようフィールドを追加する
93
+ df_with_column_number = pd.read_excel(
94
+ input_file, dtype=str, header=None, skiprows=1
95
+ )
96
+ new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
97
+ df2 = df_with_column_number.rename(columns=dict(
98
+ zip(df_with_column_number.columns, new_column_names)
99
+ ))
100
+ df = pd.concat([df, df2], axis=1)
101
+ df = df.dropna(axis=0, how='all')
102
+ df = df.dropna(axis=1, how='all')
78
103
  return df
79
104
 
80
105
  @register_loader('.json')
@@ -119,6 +144,18 @@ def save_json(
119
144
  ensure_ascii=False,
120
145
  )
121
146
 
147
+ @register_loader('.jsonl')
148
+ def load_jsonl(
149
+ input_file: str,
150
+ ):
151
+ rows = []
152
+ with open(input_file, 'r') as f:
153
+ for line in f:
154
+ row = json.loads(line)
155
+ rows.append(row)
156
+ df = pd.DataFrame(rows)
157
+ return df
158
+
122
159
  @register_saver('.jsonl')
123
160
  def save_jsonl(
124
161
  df: pd.DataFrame,
@@ -249,20 +286,29 @@ def convert(
249
286
  output_debug: bool = False,
250
287
  list_actions: list[str] | None = None,
251
288
  list_pick_columns: list[str] | None = None,
289
+ action_delimiter: str = ':',
252
290
  verbose: bool = False,
291
+ ignore_file_rows: list[str] | None = None,
253
292
  ):
254
293
  ic.enable()
255
294
  ic()
256
295
  ic(input_files)
257
296
  df_list = []
258
297
  row_list_filtered_out = []
298
+ set_ignore_file_rows = set()
259
299
  global_status = GlobalStatus()
260
300
  config = setup_config(config_path)
261
301
  ic(config)
302
+ if ignore_file_rows:
303
+ set_ignore_file_rows = set(ignore_file_rows)
262
304
  if list_pick_columns:
263
305
  setup_pick_with_args(config, list_pick_columns)
264
306
  if list_actions:
265
- setup_actions_with_args(config, list_actions)
307
+ setup_actions_with_args(
308
+ config,
309
+ list_actions,
310
+ action_delimiter=action_delimiter
311
+ )
266
312
  if output_file:
267
313
  ext = os.path.splitext(output_file)[1]
268
314
  if ext not in dict_savers:
@@ -274,6 +320,7 @@ def convert(
274
320
  ic(input_file)
275
321
  if not os.path.exists(input_file):
276
322
  raise FileNotFoundError(f'File not found: {input_file}')
323
+ base_name = os.path.basename(input_file)
277
324
  ext = os.path.splitext(input_file)[1]
278
325
  ic(ext)
279
326
  if ext not in dict_loaders:
@@ -288,10 +335,20 @@ def convert(
288
335
  #new_rows = []
289
336
  new_flat_rows = []
290
337
  for index, flat_row in df.iterrows():
338
+ file_row_index = f'{input_file}:{index}'
339
+ if file_row_index in set_ignore_file_rows:
340
+ continue
341
+ short_file_row_index = f'{base_name}:{index}'
342
+ if short_file_row_index in set_ignore_file_rows:
343
+ continue
344
+ #if flat_row.empty:
345
+ # continue
291
346
  orig_row = prepare_row(flat_row)
292
347
  row = prepare_row(flat_row)
293
348
  if STAGING_FIELD not in orig_row.nested:
294
349
  set_row_staging_value(row, FILE_FIELD, input_file)
350
+ set_row_staging_value(row, FILE_ROW_INDEX_FIELD, file_row_index)
351
+ set_row_staging_value(row, ROW_INDEX_FIELD, index)
295
352
  set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
296
353
  if config.process.assign_array:
297
354
  row.flat= assign_array(row.flat, config.process.assign_array)
@@ -300,16 +357,23 @@ def convert(
300
357
  if config.process.assign_length:
301
358
  row.flat = assign_length(row.flat, config.process.assign_length)
302
359
  if config.actions:
303
- new_row = do_actions(global_status, row, config.actions)
304
- if new_row is None:
305
- if not output_debug:
306
- pop_row_staging(row)
360
+ try:
361
+ new_row = do_actions(global_status, row, config.actions)
362
+ if new_row is None:
363
+ if not output_debug:
364
+ pop_row_staging(row)
365
+ if verbose:
366
+ ic('Filtered out: ', row.flat)
367
+ if output_file_filtered_out:
368
+ row_list_filtered_out.append(row.flat)
369
+ continue
370
+ row = new_row
371
+ except Exception as e:
307
372
  if verbose:
308
- ic('Filtered out: ', row.flat)
309
- if output_file_filtered_out:
310
- row_list_filtered_out.append(row.flat)
311
- continue
312
- row = new_row
373
+ ic(index)
374
+ ic(flat_row)
375
+ ic(row.flat)
376
+ raise e
313
377
  if config.pick:
314
378
  remap_columns(row, config.pick)
315
379
  if not output_debug:
@@ -317,6 +381,11 @@ def convert(
317
381
  new_flat_rows.append(row.flat)
318
382
  new_df = pd.DataFrame(new_flat_rows)
319
383
  df_list.append(new_df)
384
+ # NOTE: concatの仕様が変わり、all-NAの列を含むdfを連結しようとすると警告が出るようになった
385
+ #if ic(new_df.dropna(axis=1, how='all').empty):
386
+ # ic(new_df.dropna(axis=1, how='all'))
387
+ # raise ValueError('No rows to output.')
388
+ #df_list.append(new_df.dropna(axis=1, how='all'))
320
389
  all_df = pd.concat(df_list)
321
390
  #ic(all_df)
322
391
  ic(len(all_df))
@@ -325,6 +394,8 @@ def convert(
325
394
  if output_file:
326
395
  ic('Saing to: ', output_file)
327
396
  saver(all_df, output_file)
397
+ else:
398
+ ic(all_df)
328
399
  if row_list_filtered_out:
329
400
  df_filtered_out = pd.DataFrame(row_list_filtered_out)
330
401
  ic('Saving filtered out to: ', output_file_filtered_out)
@@ -0,0 +1,22 @@
1
+ # Description: Get the value of a field in a dictionary.
2
+
3
+ from collections import OrderedDict
4
+
5
+ def get_nested_field_value(
6
+ data: OrderedDict | list,
7
+ field: str,
8
+ ):
9
+ if isinstance(data, list):
10
+ if field.isdigit():
11
+ index = int(field)
12
+ if index < len(data):
13
+ return data[index], True
14
+ return None, False
15
+ if isinstance(data, dict):
16
+ if field in data:
17
+ return data[field], True
18
+ if '.' in field:
19
+ field, rest = field.split('.', 1)
20
+ if field in data:
21
+ return get_nested_field_value(data[field], rest)
22
+ return None, False
@@ -12,8 +12,13 @@ def set_nested_field_value(
12
12
  ):
13
13
  if isinstance(field, str) and '.' in field:
14
14
  field, rest = field.split('.', 1)
15
- if field not in data:
15
+ sub_data = data.get(field)
16
+ if not isinstance(sub_data, dict):
16
17
  data[field] = OrderedDict()
17
18
  set_nested_field_value(data[field], rest, value)
18
19
  else:
19
- data[field] = value
20
+ try:
21
+ data[field] = value
22
+ except:
23
+ ic(data, field, value)
24
+ raise
@@ -21,6 +21,14 @@ class Row:
21
21
  flat: OrderedDict
22
22
  nested: OrderedDict
23
23
 
24
+ @dataclasses.dataclass
25
+ class AssignConfig:
26
+ target: str
27
+ source: str
28
+ assign_default: bool = False
29
+ default_value: Any = None
30
+ required: bool = False
31
+
24
32
  @dataclasses.dataclass
25
33
  class AssignConstantConfig:
26
34
  target: str
@@ -40,7 +48,10 @@ class AssignIdConfig:
40
48
  @dataclasses.dataclass
41
49
  class FilterConfig:
42
50
  field: str
43
- operator: Literal['==', '!=', '>', '>=', '<', '<=', '=~', 'not-in']
51
+ operator: Literal[
52
+ '==', '!=', '>', '>=', '<', '<=', '=~', 'not-in',
53
+ 'empty', 'not-empty',
54
+ ]
44
55
  value: str | list[str]
45
56
 
46
57
  @dataclasses.dataclass
@@ -53,6 +64,13 @@ class JoinConfig:
53
64
  class OmitConfig:
54
65
  field: str
55
66
 
67
+ @dataclasses.dataclass
68
+ class ParseConfig:
69
+ target: str
70
+ source: str
71
+ as_type: Literal['json', 'literal']
72
+ required: bool = False
73
+
56
74
  @dataclasses.dataclass
57
75
  class PickConfig:
58
76
  target: str
@@ -1,3 +0,0 @@
1
- # Table Data Converter
2
-
3
- This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
@@ -1,2 +0,0 @@
1
- __version__ = "0.2.15"
2
- __version_tuple__ = (0, 2, 15)
@@ -1,15 +0,0 @@
1
- # Description: Get the value of a field in a dictionary.
2
-
3
- from collections import OrderedDict
4
-
5
- def get_nested_field_value(
6
- data: OrderedDict,
7
- field: str,
8
- ):
9
- if field in data:
10
- return data[field], True
11
- if '.' in field:
12
- field, rest = field.split('.', 1)
13
- if field in data:
14
- return get_nested_field_value(data[field], rest)
15
- return None, False