python-table-converter 0.2.13__tar.gz → 0.2.15__tar.gz

{python_table_converter-0.2.13 → python_table_converter-0.2.15}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.13
+Version: 0.2.15
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT
@@ -18,6 +18,7 @@ Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Requires-Dist: xlsxwriter (>=3.2.0,<4.0.0)
 Project-URL: Repository, https://github.com/akivajp/python-table-converter
 Description-Content-Type: text/markdown
 

{python_table_converter-0.2.13 → python_table_converter-0.2.15}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.13"
+version = "0.2.15"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"
@@ -26,6 +26,7 @@ tqdm = "^4.67.0"
 pandas = "^2.2.3"
 openpyxl = "^3.1.5"
 pyyaml = "^6.0.2"
+xlsxwriter = "^3.2.0"
 
 
 [build-system]

python_table_converter-0.2.15/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.15"
+__version_tuple__ = (0, 2, 15)

{python_table_converter-0.2.13 → python_table_converter-0.2.15}/table_converter/commands/convert_tables.py

@@ -12,15 +12,12 @@ def run(
     convert(
         input_files = args.input_files,
         output_file = args.output_file,
+        output_file_filtered_out = args.output_file_filtered_out,
         config_path = args.config,
-        assign_constants = args.assign_constants,
-        assign_formats = args.assign_formats,
-        str_filters = args.filters,
-        pickup_columns= args.pickup_columns,
-        fields_to_split_by_newline = args.split_by_newline,
-        fields_to_assign_ids = args.assign_ids,
-        str_omit_fields= args.omit_fields,
+        list_actions = args.do_actions,
+        list_pick_columns = args.pick_columns,
         output_debug = args.output_debug,
+        verbose = args.verbose,
     )
 
 def setup_parser(
@@ -39,44 +36,27 @@ def setup_parser(
         help='Path to the output file.'
     )
     parser.add_argument(
-        '--config', '-c',
-        type=str,
-        help='Path to the configuration file.',
-    )
-    parser.add_argument(
-        '--pickup-columns',
-        type=str,
-        help='Pickup column map',
+        '--output-file-filtered-out', '--output-filtered-out', '-f',
+        metavar='OUTPUT_FILE_FILTERED_OUT',
+        required=False,
+        help='Path to the output file for filtered out rows.',
     )
     parser.add_argument(
-        '--split-by-newline',
-        type=str,
-        help='Fields to split by newline',
-    )
-    parser.add_argument(
-        '--assign-ids',
-        type=str,
-        help='Field to assign ids',
-    )
-    parser.add_argument(
-        '--assign-constants',
-        type=str,
-        help='Field to assign constants',
-    )
-    parser.add_argument(
-        '--assign-formats',
+        '--config', '-c',
         type=str,
-        help='Field to assign formats',
+        help='Path to the configuration file.',
     )
     parser.add_argument(
-        '--filters', '--filter', '-f',
+        '--pick-columns', '--pick',
         type=str,
-        help='Expression list to filter records',
+        nargs='+',
+        help='Pick column map',
     )
     parser.add_argument(
-        '--omit-fields', '--omit',
+        '--do-actions', '--actions', '--do',
+        nargs='+',
         type=str,
-        help='Field to omit',
+        help='Actions to do',
     )
     parser.add_argument(
         '--output-debug',

python_table_converter-0.2.15/table_converter/core/actions.py

@@ -0,0 +1,423 @@
+'''
+Actions are used to transform the data in the table.
+'''
+
+from collections import OrderedDict
+from dataclasses import dataclass
+from typing import (
+    Any,
+)
+
+from icecream import ic
+
+from . config import (
+    Config,
+)
+
+from . constants import (
+    INPUT_FIELD,
+    STAGING_FIELD,
+)
+
+from . types import (
+    AssignConstantConfig,
+    AssignFormatConfig,
+    AssignIdConfig,
+    FilterConfig,
+    GlobalStatus,
+    JoinConfig,
+    OmitConfig,
+    PickConfig,
+    SplitConfig,
+    Row,
+)
+
+from . functions.assign_id import assign_id
+from . functions.flatten_row import flatten_row
+from . functions.nest_row import nest_row
+from . functions.search_column_value import search_column_value
+from . functions.set_flat_field_value import set_flat_field_value
+from . functions.set_row_value import (
+    set_row_staging_value,
+)
+from . functions.set_nested_field_value import set_nested_field_value
+
+def setup_actions_with_args(
+    config: Config,
+    list_actions: list[str],
+):
+    ic(list_actions)
+    for str_action in list_actions:
+        fields = str_action.split(':')
+        if len(fields) >= 1:
+            action_name = fields[0].strip()
+        if action_name == 'assign-format':
+            setup_assign_format_action(config, str_action)
+            continue
+        if action_name == 'filter':
+            setup_filter_action(config, str_action)
+            continue
+        if len(fields) not in [2,3]:
+            raise ValueError(
+                f'Action must have 2 or 3 colon-separated fields: {str_action}'
+            )
+        str_fields = fields[1].strip()
+        if len(fields) == 3:
+            str_options = fields[2].strip()
+        else:
+            str_options = ''
+        options = OrderedDict()
+        if str_options:
+            for str_option in str_options.split(','):
+                if '=' in str_option:
+                    key, value = str_option.split('=')
+                    options[key.strip()] = value.strip()
+                else:
+                    options[str_option.strip()] = True
+        fields = str_fields.split(',')
+        for field in fields:
+            if '=' in field:
+                target, source = field.split('=')
+                target = target.strip()
+                source = source.strip()
+            else:
+                target = field.strip()
+                source = field.strip()
+            if action_name == 'assign-constant':
+                str_type = options.get('type', 'str')
+                if str_type in ['str', 'string']:
+                    value = source
+                elif str_type in ['int', 'integer']:
+                    value = int(source)
+                elif str_type == 'float':
+                    value = float(source)
+                elif str_type in ['bool', 'boolean']:
+                    value = bool(source)
+                else:
+                    raise ValueError(
+                        f'Unsupported type: {str_type}'
+                    )
+                config.actions.append(AssignConstantConfig(
+                    target = target,
+                    value = value,
+                ))
+                continue
+            if action_name == 'assign-id':
+                context = options.get('context', None)
+                if context:
+                    context = context.split(',')
+                config.actions.append(AssignIdConfig(
+                    target = target,
+                    primary = [source],
+                    context = context,
+                ))
+                continue
+            if action_name == 'join':
+                delimiter = options.get('delimiter', None)
+                config.actions.append(JoinConfig(
+                    target = target,
+                    source = source,
+                    delimiter = delimiter,
+                ))
+                continue
+            if action_name == 'omit':
+                config.actions.append(OmitConfig(
+                    field = target,
+                ))
+                continue
+            if action_name == 'split':
+                delimiter = options.get('delimiter', None)
+                config.actions.append(SplitConfig(
+                    target = target,
+                    source = source,
+                    delimiter = delimiter,
+                ))
+                continue
+            raise ValueError(
+                f'Unsupported action: {action_name}'
+            )
+    return config
+
+def setup_assign_format_action(
+    config: Config,
+    str_action: str,
+):
+    action_fields = str_action.split(':', 1)
+    if len(action_fields) != 2:
+        raise ValueError(
+            f'Expected 2 fields separated by ":": {str_action}'
+        )
+    action_name = action_fields[0].strip()
+    assert action_name == 'assign-format'
+    assignment_fields = action_fields[1].split('=')
+    if len(assignment_fields) != 2:
+        raise ValueError(
+            f'Expected 2 fields separated by "=": {action_fields[1]}'
+        )
+    target = assignment_fields[0].strip()
+    format = assignment_fields[1].strip()
+    config.actions.append(AssignFormatConfig(
+        target = target,
+        format = format,
+    ))
+    return config
+
+def setup_filter_action(
+    config: Config,
+    str_action: str,
+):
+    action_fields = str_action.split(':', 1)
+    if len(action_fields) != 2:
+        raise ValueError(
+            f'Expected 2 fields separated by ":": {str_action}'
+        )
+    action_name = action_fields[0].strip()
+    assert action_name == 'filter'
+    str_filter = action_fields[1].strip()
+    if '==' in str_filter:
+        field, value = str_filter.split('==')
+        config.actions.append(FilterConfig(
+            field = field.strip(),
+            operator = '==',
+            value = value.strip(),
+        ))
+        return config
+    if '!=' in str_filter:
+        field, value = str_filter.split('!=')
+        config.actions.append(FilterConfig(
+            field = field.strip(),
+            operator = '!=',
+            value = value.strip(),
+        ))
+        return config
+    raise ValueError(
+        f'Unsupported filter: {str_filter}'
+    )
+
+def do_actions(
+    status: GlobalStatus,
+    row: Row,
+    actions: list[AssignConstantConfig],
+):
+    for action in actions:
+        row = do_action(status, row, action)
+        if row is None:
+            return None
+    return row
+
+def do_action(
+    status: GlobalStatus,
+    row: Row,
+    action: AssignConstantConfig,
+):
+    if isinstance(action, AssignConstantConfig):
+        return assign_constant(row, action)
+    if isinstance(action, AssignFormatConfig):
+        return assign_format(row, action)
+    if isinstance(action, AssignIdConfig):
+        return assign_id(status.id_context_map, row, action)
+    if isinstance(action, FilterConfig):
+        if filter_row(row, action):
+            return row
+        return None
+    if isinstance(action, JoinConfig):
+        return join_field(row, action)
+    if isinstance(action, OmitConfig):
+        return omit_field(row, action)
+    if isinstance(action, SplitConfig):
+        return split_field(row, action)
+    raise ValueError(
+        f'Unsupported action: {action}'
+    )
+
+def prepare_row(
+    flat_row: OrderedDict | None = None,
+):
+    if flat_row is None:
+        flat_row = OrderedDict()
+    nested_row = nest_row(flat_row)
+    return Row(
+        flat = OrderedDict(flat_row),
+        nested = nested_row,
+    )
+
+def delete_flat_row_value(
+    flat_row: OrderedDict,
+    target: str,
+):
+    prefix = f'{target}.'
+    for key in list(flat_row.keys()):
+        if key == target or key.startswith(prefix):
+            del flat_row[key]
+
+def pop_nested_row_value(
+    nested_row: OrderedDict,
+    key: str,
+    default: Any = None,
+):
+    keys = key.split('.')
+    for key in keys[:-1]:
+        if key not in nested_row:
+            return default, False
+        nested_row = nested_row[key]
+    return nested_row.pop(keys[-1], default), True
+
+def pop_row_value(
+    row: Row,
+    key: str,
+    default: Any = None,
+):
+    delete_flat_row_value(row.flat, key)
+    return pop_nested_row_value(row.nested, key, default)
+
+def pop_row_staging(
+    row: Row,
+    default: Any = None,
+):
+    return pop_row_value(row, STAGING_FIELD, default)
+
+def assign_constant(
+    row: Row,
+    config: AssignConstantConfig,
+):
+    set_row_staging_value(row, config.target, config.value)
+    return row
+
+def split_field(
+    row: Row,
+    config: SplitConfig,
+):
+    value, found = search_column_value(row.flat, config.source)
+    if found:
+        if isinstance(value, str):
+            new_value = value.split(config.delimiter)
+            new_value = list(filter(None, new_value))
+            value = new_value
+        set_row_staging_value(row, config.target, value)
+    return row
+
+def remap_columns(
+    row: Row,
+    list_config: list[PickConfig],
+):
+    if not list_config:
+        list_config = []
+        for key in row.nested[STAGING_FIELD][INPUT_FIELD].keys():
+            list_config.append(PickConfig(
+                source = key,
+                target = key,
+            ))
+    new_flat_row = OrderedDict()
+    picked = []
+    for config in list_config:
+        value, key = search_column_value(row.nested, config.source)
+        if key:
+            set_flat_field_value(new_flat_row, config.target, value)
+            picked.append(key)
+    for key in row.flat.keys():
+        if key in picked:
+            if not key.startswith(f'{STAGING_FIELD}.{INPUT_FIELD}.'):
+                continue
+        if key in new_flat_row:
+            continue
+        if key.startswith(f'{STAGING_FIELD}.'):
+            # NOTE: Skip staging fields
+            new_flat_row[key] = row.flat[key]
+        else:
+            input_key = f'{STAGING_FIELD}.{INPUT_FIELD}.{key}'
+            if input_key in row.flat:
+                value = row.flat[key]
+                input_value = row.flat[input_key]
+                if value == input_value:
+                    # NOTE: Skip if the same value in the input field
+                    continue
+            # NOTE: Set the unused value to the staging field
+            new_flat_row[f'{STAGING_FIELD}.{key}'] = row.flat[key]
+    row.flat = new_flat_row
+    row.nested = nest_row(new_flat_row)
+    return row
+
+def assign_format(
+    row: Row,
+    config: AssignFormatConfig,
+):
+    template = config.format
+    params = {}
+    for key, value in row.flat.items():
+        for prefix in [
+            f'{STAGING_FIELD}.{INPUT_FIELD}.',
+            f'{STAGING_FIELD}.',
+        ]:
+            if key.startswith(prefix):
+                rest = key[len(prefix):]
+                params[rest] = value
+    params.update(row.flat)
+    formatted = None
+    while formatted is None:
+        try:
+            formatted = template.format(**params)
+        except KeyError as e:
+            #ic(e)
+            #ic(e.args)
+            #ic(e.args[0])
+            key = e.args[0]
+            params[key] = f'__{key}__undefined__'
+        except:
+            #ic(params)
+            ic(params.keys())
+            raise
+    set_row_staging_value(row, config.target, formatted)
+    return row
+
+def filter_row(
+    row: Row,
+    config: list[FilterConfig],
+):
+    value, found = search_column_value(row.nested, config.field)
+    #ic(config, value, found)
+    if config.operator == '==':
+        if not found:
+            return False
+        if value != config.value and str(value) != str(config.value):
+            return False
+    elif config.operator == '!=':
+        if str(value) == str(config.value) or value == config.value:
+            return False
+    elif config.operator == 'not-in':
+        if isinstance(config.value, list):
+            if value in config.value:
+                return False
+            if str(value) in config.value:
+                return False
+        else:
+            raise ValueError(f'Unsupported filter value type: type{config.value}')
+    else:
+        raise ValueError(f'Unsupported operator: {config.operator}')
+    return True
+
+def omit_field(
+    row: Row,
+    config: OmitConfig,
+):
+    value, found = pop_row_value(row, config.field)
+    if not found:
+        return row
+    if f'{STAGING_FIELD}.{config.field}' not in row.flat:
+        set_row_staging_value(row, config.field, value)
+    return row
+
+def join_field(
+    row: Row,
+    config: JoinConfig,
+):
+    value, found = search_column_value(row.nested, config.source)
+    if found:
+        delimiter = config.delimiter
+        if delimiter is None:
+            delimiter = ';'
+        if delimiter == '\\n':
+            delimiter = '\n'
+        if isinstance(value, list):
+            value = delimiter.join(value)
+        set_row_staging_value(row, config.target, value)
+    return row