python-table-converter 0.2.17__tar.gz → 0.3.0__tar.gz

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.17
+Version: 0.3.0
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT
@@ -22,6 +22,21 @@ Requires-Dist: xlsxwriter (>=3.2.0,<4.0.0)
 Project-URL: Repository, https://github.com/akivajp/python-table-converter
 Description-Content-Type: text/markdown
 
+# ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
+
+This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
+The Python package is now distributed as `tabpro` on PyPI.
+
+```sh
+pip install tabpro
+```
+
+## Migration Repository
+- Repository: [TabPro](https://github.com/akivajp/tabpro)
+- PyPI package: [tabpro](https://pypi.org/project/tabpro/)
+
+---
+
 # Table Data Converter
 
 This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.

python_table_converter-0.3.0/README.md

@@ -0,0 +1,18 @@
+# ⚠️ DEPRECATED: This project has been migrated to [TabPro](https://github.com/akivajp/tabpro)
+
+This repository is no longer maintained. Please use [TabPro](https://github.com/akivajp/tabpro) instead.
+The Python package is now distributed as `tabpro` on PyPI.
+
+```sh
+pip install tabpro
+```
+
+## Migration Repository
+- Repository: [TabPro](https://github.com/akivajp/tabpro)
+- PyPI package: [tabpro](https://pypi.org/project/tabpro/)
+
+---
+
+# Table Data Converter
+
+This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.17"
+version = "0.3.0"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"
@@ -11,12 +11,12 @@ packages = [
 ]
 
 [tool.poetry-dynamic-versioning]
-enable = false
+enable = true
 style = "pep440"
 
 [tool.poetry.scripts]
 table-converter = "table_converter.cli:main"
-conver-tables = "table_converter.cli:command_convert_tables"
+convert-tables = "table_converter.cli:command_convert_tables"
 
 [tool.poetry.dependencies]
 python = "^3.10"

python_table_converter-0.3.0/table_converter/__init__.py

@@ -0,0 +1,16 @@
+# 旧パッケージは TabPro (https://github.com/akivajp/tabpro) に移行済み。
+# import・CLI 実行時に移行を促す警告を表示する。
+# ※ DeprecationWarning は Python のデフォルトフィルタで非表示になるため、
+#   デフォルトで表示される FutureWarning を使用している。
+import warnings
+
+warnings.warn(
+    "The 'python-table-converter' package has been renamed to 'tabpro'. "
+    "Please install the new package instead: pip install tabpro "
+    "(https://github.com/akivajp/tabpro)",
+    FutureWarning,
+    stacklevel=2,
+)
+
+__version__ = "0.3.0"
+__version_tuple__ = (0, 3, 0)

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/commands/convert_tables.py

@@ -16,8 +16,10 @@ def run(
         config_path = args.config,
         list_actions = args.do_actions,
         list_pick_columns = args.pick_columns,
+        action_delimiter = args.action_delimiter,
         output_debug = args.output_debug,
         verbose = args.verbose,
+        ignore_file_rows = args.ignore_file_rows,
     )
 
 def setup_parser(
@@ -30,9 +32,9 @@ def setup_parser(
         help='Path to the input file.'
     )
     parser.add_argument(
-        '--output-file', '-o',
+        '--output-file', '--output', '-o',
         metavar='OUTPUT_FILE',
-        required=True,
+        required=False,
         help='Path to the output file.'
     )
     parser.add_argument(
@@ -52,12 +54,24 @@ def setup_parser(
         nargs='+',
         help='Pick column map',
     )
+    parser.add_argument(
+        '--action-delimiter', '--do-delimiter', '--do-delim',
+        type=str,
+        default=':',
+        help='Action delimiter',
+    )
     parser.add_argument(
         '--do-actions', '--actions', '--do',
         nargs='+',
         type=str,
         help='Actions to do',
     )
+    parser.add_argument(
+        '--ignore-file-rows', '--ignore-rows', '--ignore',
+        nargs='+',
+        type=str,
+        help='Ignore tuples of file name and row index',
+    )
     parser.add_argument(
         '--output-debug',
         action='store_true',

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/core/actions.py

@@ -2,6 +2,8 @@
 Actions are used to transform the data in the table.
 '''
 
+import ast
+import json
 import re
 
 from collections import OrderedDict
@@ -22,6 +24,7 @@ from . constants import (
 )
 
 from . types import (
+    AssignConfig,
     AssignConstantConfig,
     AssignFormatConfig,
     AssignIdConfig,
@@ -29,6 +32,7 @@ from . types import (
     GlobalStatus,
     JoinConfig,
     OmitConfig,
+    ParseConfig,
     PickConfig,
     SplitConfig,
     Row,
@@ -47,21 +51,23 @@ from . functions.set_nested_field_value import set_nested_field_value
 def setup_actions_with_args(
     config: Config,
     list_actions: list[str],
+    action_delimiter: str = ':',
 ):
     ic(list_actions)
     for str_action in list_actions:
-        fields = str_action.split(':')
+        fields = str_action.split(action_delimiter)
         if len(fields) >= 1:
             action_name = fields[0].strip()
         if action_name == 'assign-format':
-            setup_assign_format_action(config, str_action)
+            setup_assign_format_action(config, str_action, action_delimiter)
             continue
         if action_name == 'filter':
-            setup_filter_action(config, str_action)
+            setup_filter_action(config, str_action, action_delimiter)
             continue
         if len(fields) not in [2,3]:
             raise ValueError(
-                f'Action must have 2 or 3 colon-separated fields: {str_action}'
+                'Action must have 2 or 3 delimiter-separated fields: ' +
+                f'delimiter:{action_delimiter!r}, action string: {str_action!r}'
             )
         str_fields = fields[1].strip()
         if len(fields) == 3:
@@ -85,6 +91,23 @@ def setup_actions_with_args(
             else:
                 target = field.strip()
                 source = field.strip()
+            if action_name == 'assign':
+                assign_default = False
+                default_value = None
+                if 'default' in options:
+                    assign_default = True
+                    default_value = options['default']
+                if default_value in ['None', 'none', 'Null', 'null']:
+                    default_value = None
+                required = options.get('required', False)
+                config.actions.append(AssignConfig(
+                    target = target,
+                    source = source,
+                    assign_default = assign_default,
+                    default_value = default_value,
+                    required = required,
+                ))
+                continue
             if action_name == 'assign-constant':
                 str_type = options.get('type', 'str')
                 if str_type in ['str', 'string']:
@@ -114,6 +137,20 @@ def setup_actions_with_args(
                     context = context,
                 ))
                 continue
+            if action_name == 'filter-empty':
+                config.actions.append(FilterConfig(
+                    field = target,
+                    operator = 'empty',
+                    value = '',
+                ))
+                continue
+            if action_name == 'filter-not-empty':
+                config.actions.append(FilterConfig(
+                    field = target,
+                    operator = 'not-empty',
+                    value = '',
+                ))
+                continue
             if action_name == 'join':
                 delimiter = options.get('delimiter', None)
                 config.actions.append(JoinConfig(
@@ -127,8 +164,33 @@ def setup_actions_with_args(
                     field = target,
                 ))
                 continue
+            if action_name == 'parse':
+                as_type = options.get('as', 'literal')
+                required = options.get('required', False)
+                if as_type not in ['json', 'literal']:
+                    raise ValueError(
+                        f'Unsupported as type: {as_type}'
+                    )
+                config.actions.append(ParseConfig(
+                    target = target,
+                    source = source,
+                    as_type = as_type,
+                    required = required,
+                ))
+                continue
+            if action_name == 'parse-json':
+                required = options.get('required', False)
+                config.actions.append(ParseConfig(
+                    target = target,
+                    source = source,
+                    as_type = 'json',
+                    required = required,
+                ))
+                continue
             if action_name == 'split':
                 delimiter = options.get('delimiter', None)
+                if delimiter == '\\n':
+                    delimiter = '\n'
                 config.actions.append(SplitConfig(
                     target = target,
                     source = source,
@@ -143,8 +205,9 @@ def setup_actions_with_args(
 def setup_assign_format_action(
     config: Config,
     str_action: str,
+    delimiter: str = ':',
 ):
-    action_fields = str_action.split(':', 1)
+    action_fields = str_action.split(delimiter, 1)
     if len(action_fields) != 2:
         raise ValueError(
             f'Expected 2 fields separated by ":": {str_action}'
@@ -167,8 +230,9 @@ def setup_assign_format_action(
 def setup_filter_action(
     config: Config,
     str_action: str,
+    delimiter: str = ':',
 ):
-    action_fields = str_action.split(':', 1)
+    action_fields = str_action.split(delimiter, 1)
     if len(action_fields) != 2:
         raise ValueError(
             f'Expected 2 fields separated by ":": {str_action}'
@@ -220,6 +284,8 @@ def do_action(
     row: Row,
     action: AssignConstantConfig,
 ):
+    if isinstance(action, AssignConfig):
+        return assign(row, action)
     if isinstance(action, AssignConstantConfig):
         return assign_constant(row, action)
     if isinstance(action, AssignFormatConfig):
@@ -232,6 +298,8 @@ def do_action(
         return None
     if isinstance(action, JoinConfig):
         return join_field(row, action)
+    if isinstance(action, ParseConfig):
+        return parse(row, action)
     if isinstance(action, OmitConfig):
         return omit_field(row, action)
     if isinstance(action, SplitConfig):
@@ -245,7 +313,11 @@ def prepare_row(
 ):
     if flat_row is None:
         flat_row = OrderedDict()
-    nested_row = nest_row(flat_row)
+    try:
+        nested_row = nest_row(flat_row)
+    except:
+        ic(flat_row)
+        raise
     return Row(
         flat = OrderedDict(flat_row),
         nested = nested_row,
@@ -301,6 +373,7 @@ def split_field(
     if found:
         if isinstance(value, str):
             new_value = value.split(config.delimiter)
+            new_value = map(str.strip, new_value)
             new_value = list(filter(None, new_value))
             value = new_value
         set_row_staging_value(row, config.target, value)
@@ -347,6 +420,47 @@ def remap_columns(
     row.nested = nest_row(new_flat_row)
     return row
 
+
+def search_with_operator(
+    row: Row,
+    source: str,
+):
+    or_operator = '\|\|'
+    null_or_operator = '\?\?'
+    operator_group = f'{or_operator}|{null_or_operator}'
+    matched = re.split(f'({operator_group})', source, 1)
+    #ic(source, matched)
+    if len(matched) == 1:
+        return search_column_value(row.nested, source)
+    matched = map(str.strip, matched)
+    left, operator, rest = matched
+    value, found = search_column_value(row.nested, left)
+    if operator == '||':
+        if bool(value):
+            return value, found
+    if operator == '??':
+        if found and value is not None:
+            return value, found
+    return search_with_operator(row, rest)
+
+def assign(
+    row: Row,
+    config: AssignConfig,
+):
+    value, found = search_with_operator(row, config.source)
+    if config.required:
+        if not found or bool(value) == False:
+            raise ValueError(
+                'Required field not found or empty, ' +
+                f'field: {config.source}, found: {found}, value: {value}'
+            )
+    if found:
+        set_row_staging_value(row, config.target, value)
+    else:
+        if config.assign_default:
+            set_row_staging_value(row, config.target, config.default_value)
+    return row
+
 def assign_format(
     row: Row,
     config: AssignFormatConfig,
@@ -379,6 +493,14 @@ def assign_format(
     set_row_staging_value(row, config.target, formatted)
     return row
 
+def check_empty(
+    value: Any,
+    found: str | None,
+):
+    if not found:
+        return True
+    return not bool(value)
+
 def filter_row(
     row: Row,
     config: list[FilterConfig],
@@ -406,6 +528,12 @@ def filter_row(
                 return False
         else:
             raise ValueError(f'Unsupported filter value type: type{config.value}')
+    elif config.operator == 'empty':
+        if not check_empty(value, found):
+            return False
+    elif config.operator == 'not-empty':
+        if check_empty(value, found):
+            return False
     else:
         raise ValueError(f'Unsupported operator: {config.operator}')
     return True
@@ -436,3 +564,38 @@ def join_field(
             value = delimiter.join(value)
         set_row_staging_value(row, config.target, value)
     return row
+
+def parse(
+    row: Row,
+    config: AssignConfig,
+):
+    value, found = search_column_value(row.nested, config.source)
+    if config.required:
+        if not found:
+            raise ValueError(
+                f'Required field not found, field: {config.source}'
+            )
+    if found:
+        if type(value) == str:
+            if config.as_type == 'literal':
+                try:
+                    parsed = ast.literal_eval(value)
+                except:
+                    raise ValueError(
+                        f'Failed to parse literal: {value}'
+                    )
+            elif config.as_type == 'json':
+                try:
+                    parsed = json.loads(value)
+                except:
+                    raise ValueError(
+                        f'Failed to parse JSON: {value}'
+                    )
+            else:
+                raise ValueError(
+                    f'Unsupported as type: {config.as_type}'
+                )
+        else:
+            parsed = value
+        set_row_staging_value(row, config.target, parsed)
+    return row

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/core/constants.py

@@ -1,3 +1,5 @@
 FILE_FIELD = '__file__'
+ROW_INDEX_FIELD = '__row_index__'
+FILE_ROW_INDEX_FIELD = '__file_row_index__'
 INPUT_FIELD = '__input__'
 STAGING_FIELD = '__staging__'

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/core/convert.py

@@ -26,6 +26,8 @@ from . config import (
 )
 from . constants import (
     FILE_FIELD,
+    ROW_INDEX_FIELD,
+    FILE_ROW_INDEX_FIELD,
     INPUT_FIELD,
     STAGING_FIELD,
 )
@@ -70,11 +72,34 @@ def register_saver(
         return saver
     return decorator
 
+@register_loader('.csv')
+def load_csv(
+    input_file: str,
+):
+    # utf-8
+    #df = pd.read_csv(input_file)
+    # UTF-8 with BOM
+    df = pd.read_csv(input_file, encoding='utf-8-sig')
+    return df
+
 @register_loader('.xlsx')
 def load_excel(
     input_file: str,
 ):
-    df = pd.read_excel(input_file)
+    #df = pd.read_excel(input_file)
+    # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
+    df = pd.read_excel(input_file, dtype=str)
+    # NOTE: 列番号でもアクセスできるようフィールドを追加する
+    df_with_column_number = pd.read_excel(
+        input_file, dtype=str, header=None, skiprows=1
+    )
+    new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
+    df2 = df_with_column_number.rename(columns=dict(
+        zip(df_with_column_number.columns, new_column_names)
+    ))
+    df = pd.concat([df, df2], axis=1)
+    df = df.dropna(axis=0, how='all')
+    df = df.dropna(axis=1, how='all')
     return df
 
 @register_loader('.json')
@@ -261,20 +286,29 @@ def convert(
     output_debug: bool = False,
     list_actions: list[str] | None = None,
     list_pick_columns: list[str] | None = None,
+    action_delimiter: str = ':',
     verbose: bool = False,
+    ignore_file_rows: list[str] | None = None,
 ):
     ic.enable()
     ic()
     ic(input_files)
     df_list = []
     row_list_filtered_out = []
+    set_ignore_file_rows = set()
     global_status = GlobalStatus()
     config = setup_config(config_path)
     ic(config)
+    if ignore_file_rows:
+        set_ignore_file_rows = set(ignore_file_rows)
     if list_pick_columns:
         setup_pick_with_args(config, list_pick_columns)
     if list_actions:
-        setup_actions_with_args(config, list_actions)
+        setup_actions_with_args(
+            config,
+            list_actions,
+            action_delimiter=action_delimiter
+        )
     if output_file:
         ext = os.path.splitext(output_file)[1]
         if ext not in dict_savers:
@@ -286,6 +320,7 @@ def convert(
         ic(input_file)
         if not os.path.exists(input_file):
             raise FileNotFoundError(f'File not found: {input_file}')
+        base_name = os.path.basename(input_file)
         ext = os.path.splitext(input_file)[1]
         ic(ext)
         if ext not in dict_loaders:
@@ -300,10 +335,20 @@ def convert(
         #new_rows = []
         new_flat_rows = []
         for index, flat_row in df.iterrows():
+            file_row_index = f'{input_file}:{index}'
+            if file_row_index in set_ignore_file_rows:
+                continue
+            short_file_row_index = f'{base_name}:{index}'
+            if short_file_row_index in set_ignore_file_rows:
+                continue
+            #if flat_row.empty:
+            #    continue
             orig_row = prepare_row(flat_row)
             row = prepare_row(flat_row)
             if STAGING_FIELD not in orig_row.nested:
                 set_row_staging_value(row, FILE_FIELD, input_file)
+                set_row_staging_value(row, FILE_ROW_INDEX_FIELD, file_row_index)
+                set_row_staging_value(row, ROW_INDEX_FIELD, index)
                 set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
             if config.process.assign_array:
                 row.flat= assign_array(row.flat, config.process.assign_array)
@@ -312,16 +357,23 @@ def convert(
             if config.process.assign_length:
                 row.flat = assign_length(row.flat, config.process.assign_length)
             if config.actions:
-                new_row = do_actions(global_status, row, config.actions)
-                if new_row is None:
-                    if not output_debug:
-                        pop_row_staging(row)
+                try:
+                    new_row = do_actions(global_status, row, config.actions)
+                    if new_row is None:
+                        if not output_debug:
+                            pop_row_staging(row)
+                        if verbose:
+                            ic('Filtered out: ', row.flat)
+                        if output_file_filtered_out:
+                            row_list_filtered_out.append(row.flat)
+                        continue
+                    row = new_row
+                except Exception as e:
                     if verbose:
-                        ic('Filtered out: ', row.flat)
-                    if output_file_filtered_out:
-                        row_list_filtered_out.append(row.flat)
-                    continue
-                row = new_row
+                        ic(index)
+                        ic(flat_row)
+                        ic(row.flat)
+                    raise e
             if config.pick:
                 remap_columns(row, config.pick)
             if not output_debug:
@@ -329,6 +381,11 @@ def convert(
             new_flat_rows.append(row.flat)
         new_df = pd.DataFrame(new_flat_rows)
         df_list.append(new_df)
+        # NOTE: concatの仕様が変わり、all-NAの列を含むdfを連結しようとすると警告が出るようになった
+        #if ic(new_df.dropna(axis=1, how='all').empty):
+        #    ic(new_df.dropna(axis=1, how='all'))
+        #    raise ValueError('No rows to output.')
+        #df_list.append(new_df.dropna(axis=1, how='all'))
     all_df = pd.concat(df_list)
     #ic(all_df)
     ic(len(all_df))
@@ -337,6 +394,8 @@ def convert(
     if output_file:
         ic('Saing to: ', output_file)
         saver(all_df, output_file)
+    else:
+        ic(all_df)
     if row_list_filtered_out:
         df_filtered_out = pd.DataFrame(row_list_filtered_out)
         ic('Saving filtered out to: ', output_file_filtered_out)

python_table_converter-0.3.0/table_converter/core/functions/get_nested_field_value.py

@@ -0,0 +1,22 @@
+# Description: Get the value of a field in a dictionary.
+
+from collections import OrderedDict
+
+def get_nested_field_value(
+    data: OrderedDict | list,
+    field: str,
+):
+    if isinstance(data, list):
+        if field.isdigit():
+            index = int(field)
+            if index < len(data):
+                return data[index], True
+            return None, False
+    if isinstance(data, dict):
+        if field in data:
+            return data[field], True
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field in data:
+            return get_nested_field_value(data[field], rest)
+    return None, False

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/core/functions/set_nested_field_value.py

@@ -12,8 +12,13 @@ def set_nested_field_value(
 ):
     if isinstance(field, str) and  '.' in field:
         field, rest = field.split('.', 1)
-        if field not in data:
+        sub_data = data.get(field)
+        if not isinstance(sub_data, dict):
             data[field] = OrderedDict()
         set_nested_field_value(data[field], rest, value)
     else:
-        data[field] = value
+        try:
+            data[field] = value
+        except:
+            ic(data, field, value)
+            raise

{python_table_converter-0.2.17 → python_table_converter-0.3.0}/table_converter/core/types.py

@@ -21,6 +21,14 @@ class Row:
     flat: OrderedDict
     nested: OrderedDict
 
+@dataclasses.dataclass
+class AssignConfig:
+    target: str
+    source: str
+    assign_default: bool = False
+    default_value: Any = None
+    required: bool = False
+
 @dataclasses.dataclass
 class AssignConstantConfig:
     target: str
@@ -40,7 +48,10 @@ class AssignIdConfig:
 @dataclasses.dataclass
 class FilterConfig:
     field: str
-    operator: Literal['==', '!=', '>', '>=', '<', '<=', '=~', 'not-in']
+    operator: Literal[
+        '==', '!=', '>', '>=', '<', '<=', '=~', 'not-in',
+        'empty', 'not-empty',
+    ]
     value: str | list[str]
 
 @dataclasses.dataclass
@@ -53,6 +64,13 @@ class JoinConfig:
 class OmitConfig:
     field: str
 
+@dataclasses.dataclass
+class ParseConfig:
+    target: str
+    source: str
+    as_type: Literal['json', 'literal']
+    required: bool = False
+
 @dataclasses.dataclass
 class PickConfig:
     target: str

python_table_converter-0.2.17/README.md

@@ -1,3 +0,0 @@
-# Table Data Converter
-
-This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.

python_table_converter-0.2.17/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.17"
-__version_tuple__ = (0, 2, 17)

python_table_converter-0.2.17/table_converter/core/functions/get_nested_field_value.py

@@ -1,15 +0,0 @@
-# Description: Get the value of a field in a dictionary.
-
-from collections import OrderedDict
-
-def get_nested_field_value(
-    data: OrderedDict,
-    field: str,
-):
-    if field in data:
-        return data[field], True
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field in data:
-            return get_nested_field_value(data[field], rest)
-    return None, False