python-table-processor 0.2.29__py3-none-any.whl

table_processor/core/convert.py

@@ -0,0 +1,402 @@
+# -*- coding: utf-8 -*-
+
+import json
+import math
+import os
+
+from collections import OrderedDict
+
+from typing import (
+    Mapping,
+)
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+
+# local
+
+from . config import (
+    AssignArrayConfig,
+    PushConfig,
+    setup_config,
+    setup_pick_with_args,
+)
+from . constants import (
+    FILE_FIELD,
+    ROW_INDEX_FIELD,
+    FILE_ROW_INDEX_FIELD,
+    INPUT_FIELD,
+    STAGING_FIELD,
+)
+from . functions.flatten_row import flatten_row
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.nest_row import nest_row as nest
+from . functions.search_column_value import search_column_value
+from . functions.set_nested_field_value import set_nested_field_value
+from . functions.set_row_value import (
+    set_row_value,
+    set_row_staging_value,
+)
+
+from . actions import (
+    do_actions,
+    pop_row_staging,
+    prepare_row,
+    remap_columns,
+    setup_actions_with_args,
+)
+
+from . types import (
+    GlobalStatus,
+)
+
+dict_loaders: dict[str, callable] = {}
+def register_loader(
+    ext: str,
+):
+    def decorator(loader):
+        dict_loaders[ext] = loader
+        return loader
+    return decorator
+
+dict_savers: dict[str, callable] = {}
+def register_saver(
+    ext: str,
+):
+    def decorator(saver):
+        dict_savers[ext] = saver
+        return saver
+    return decorator
+
+@register_loader('.csv')
+def load_csv(
+    input_file: str,
+):
+    # utf-8
+    #df = pd.read_csv(input_file)
+    # UTF-8 with BOM
+    df = pd.read_csv(input_file, encoding='utf-8-sig')
+    return df
+
+@register_loader('.xlsx')
+def load_excel(
+    input_file: str,
+):
+    #df = pd.read_excel(input_file)
+    # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
+    df = pd.read_excel(input_file, dtype=str)
+    # NOTE: 列番号でもアクセスできるようフィールドを追加する
+    df_with_column_number = pd.read_excel(
+        input_file, dtype=str, header=None, skiprows=1
+    )
+    new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
+    df2 = df_with_column_number.rename(columns=dict(
+        zip(df_with_column_number.columns, new_column_names)
+    ))
+    df = pd.concat([df, df2], axis=1)
+    df = df.dropna(axis=0, how='all')
+    df = df.dropna(axis=1, how='all')
+    return df
+
+@register_loader('.json')
+def load_json(
+    input_file: str,
+):
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f'Invalid JSON array data: {input_file}')
+    #ic(data[0])
+    rows = []
+    for row in data:
+        new_row = flatten_row(row)
+        rows.append(new_row)
+    df = pd.DataFrame(rows)
+    return df
+
+@register_saver('.json')
+def save_json(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    force_ascii=False,
+    #    indent=2,
+    #    escape_forward_slashes=False,
+    #)
+    #ic(df.iloc[0])
+    data = df.to_dict(orient='records')
+    #ic(data[0])
+    data = [nest(row) for row in data]
+    #ic(data[0])
+    with open(output_file, 'w') as f:
+        json.dump(
+            data,
+            f,
+            indent=2,
+            ensure_ascii=False,
+        )
+
+@register_loader('.jsonl')
+def load_jsonl(
+    input_file: str,
+):
+    rows = []
+    with open(input_file, 'r') as f:
+        for line in f:
+            row = json.loads(line)
+            rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
+@register_saver('.jsonl')
+def save_jsonl(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    lines=True,
+    #    force_ascii=False,
+    #)
+    with open(output_file, 'w') as f:
+        for index, row in df.iterrows():
+            data = row.to_dict()
+            json.dump(
+                data,
+                f,
+                ensure_ascii=False,
+            )
+            f.write('\n')
+
+@register_saver('.csv')
+def save_csv(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # utf-8
+    #df.to_csv(output_file, index=False)
+    # UTF-8 with BOM
+    df.to_csv(output_file, index=False, encoding='utf-8-sig')
+
+@register_saver('.xlsx')
+def save_excel(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # openpyxl
+    df.to_excel(output_file, index=False)
+    # xlsxwriter
+    #writer = pd.ExcelWriter(
+    #    output_file,
+    #    engine='xlsxwriter',
+    #    engine_kwargs={
+    #        'options': {
+    #            'strings_to_urls': False,
+    #        },
+    #    }
+    #)
+    #df.to_excel(writer, index=False)
+    #writer.close()
+
+def assign_array(
+    row: OrderedDict,
+    dict_config: Mapping[str, list[AssignArrayConfig]],
+):
+    new_row = OrderedDict(row)
+    #ic(dict_config)
+    for key, config in dict_config.items():
+        array = []
+        for item in config:
+            value, found = search_column_value(row, item.field)
+            if found and value is not None:
+                array.append(value)
+            elif not item.optional:
+                array.append(None)
+        new_row[f'{STAGING_FIELD}.{key}'] = array
+    return new_row
+
+def search_column_value_from_nested(
+    nested_row: OrderedDict,
+    column: str,
+):
+    if STAGING_FIELD in nested_row:
+        value, found = get_nested_field_value(nested_row[STAGING_FIELD], column)
+        if found:
+            return value, True
+    value, found = get_nested_field_value(nested_row[STAGING_FIELD], column)
+    original, found = get_nested_field_value(nested_row, f'{STAGING_FIELD}.{INPUT_FIELD}')
+    if found:
+        value, found = get_nested_field_value(original, column)
+        if found:
+            return value, True
+    value, found = get_nested_field_value(nested_row, column)
+    if found:
+        return value, True
+    return None, False
+
+
+def push_fields(
+    row: OrderedDict,
+    list_config: list[PushConfig],
+):
+    nested_row = nest(row)
+    for config in list_config:
+        target_value, found = search_column_value_from_nested(nested_row, config.target)
+        if found:
+            array = target_value
+        else:
+            array = []
+            #set_field_value(nested_row, f'{STAGING_FIELD}.{config.target}', array)
+            set_row_staging_value(nested_row, config.target, array)
+        source_value, found = search_column_value_from_nested(nested_row, config.source)
+        if config.condition is None:
+            array.append(source_value)
+            continue
+        condition_value, found = search_column_value_from_nested(nested_row, config.condition)
+        if condition_value:
+            array.append(source_value)
+    return flatten_row(nested_row)
+
+def assign_length(
+    row: OrderedDict,
+    dict_fields: OrderedDict,
+):
+    new_row = OrderedDict(row)
+    for key, field in dict_fields.items():
+        value, found = search_column_value(row, field)
+        if found:
+            new_row[f'{STAGING_FIELD}.{key}'] = len(value)
+    return new_row
+
+def convert(
+    input_files: list[str],
+    output_file: str | None = None,
+    output_file_filtered_out: str | None = None,
+    config_path: str | None = None,
+    output_debug: bool = False,
+    list_actions: list[str] | None = None,
+    list_pick_columns: list[str] | None = None,
+    action_delimiter: str = ':',
+    verbose: bool = False,
+    ignore_file_rows: list[str] | None = None,
+):
+    ic.enable()
+    ic()
+    ic(input_files)
+    df_list = []
+    row_list_filtered_out = []
+    set_ignore_file_rows = set()
+    global_status = GlobalStatus()
+    config = setup_config(config_path)
+    ic(config)
+    if ignore_file_rows:
+        set_ignore_file_rows = set(ignore_file_rows)
+    if list_pick_columns:
+        setup_pick_with_args(config, list_pick_columns)
+    if list_actions:
+        setup_actions_with_args(
+            config,
+            list_actions,
+            action_delimiter=action_delimiter
+        )
+    if output_file:
+        ext = os.path.splitext(output_file)[1]
+        if ext not in dict_savers:
+            raise ValueError(f'Unsupported file type: {ext}')
+        saver = dict_savers[ext]
+    ic(config)
+    #return # debug return
+    for input_file in input_files:
+        ic(input_file)
+        if not os.path.exists(input_file):
+            raise FileNotFoundError(f'File not found: {input_file}')
+        base_name = os.path.basename(input_file)
+        ext = os.path.splitext(input_file)[1]
+        ic(ext)
+        if ext not in dict_loaders:
+            raise ValueError(f'Unsupported file type: {ext}')
+        df = dict_loaders[ext](input_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        #ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        #new_rows = []
+        new_flat_rows = []
+        for index, flat_row in df.iterrows():
+            file_row_index = f'{input_file}:{index}'
+            if file_row_index in set_ignore_file_rows:
+                continue
+            short_file_row_index = f'{base_name}:{index}'
+            if short_file_row_index in set_ignore_file_rows:
+                continue
+            #if flat_row.empty:
+            #    continue
+            orig_row = prepare_row(flat_row)
+            row = prepare_row(flat_row)
+            if STAGING_FIELD not in orig_row.nested:
+                set_row_staging_value(row, FILE_FIELD, input_file)
+                set_row_staging_value(row, FILE_ROW_INDEX_FIELD, file_row_index)
+                set_row_staging_value(row, ROW_INDEX_FIELD, index)
+                set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
+            if config.process.assign_array:
+                row.flat= assign_array(row.flat, config.process.assign_array)
+            if config.process.push:
+                row.flat = push_fields(row.flat, config.process.push)
+            if config.process.assign_length:
+                row.flat = assign_length(row.flat, config.process.assign_length)
+            if config.actions:
+                try:
+                    new_row = do_actions(global_status, row, config.actions)
+                    if new_row is None:
+                        if not output_debug:
+                            pop_row_staging(row)
+                        if verbose:
+                            ic('Filtered out: ', row.flat)
+                        if output_file_filtered_out:
+                            row_list_filtered_out.append(row.flat)
+                        continue
+                    row = new_row
+                except Exception as e:
+                    if verbose:
+                        ic(index)
+                        ic(flat_row)
+                        ic(row.flat)
+                    raise e
+            if config.pick:
+                remap_columns(row, config.pick)
+            if not output_debug:
+                pop_row_staging(row)
+            new_flat_rows.append(row.flat)
+        new_df = pd.DataFrame(new_flat_rows)
+        df_list.append(new_df)
+        # NOTE: concatの仕様が変わり、all-NAの列を含むdfを連結しようとすると警告が出るようになった
+        #if ic(new_df.dropna(axis=1, how='all').empty):
+        #    ic(new_df.dropna(axis=1, how='all'))
+        #    raise ValueError('No rows to output.')
+        #df_list.append(new_df.dropna(axis=1, how='all'))
+    all_df = pd.concat(df_list)
+    #ic(all_df)
+    ic(len(all_df))
+    #ic(all_df.columns)
+    #ic(all_df.iloc[0])
+    if output_file:
+        ic('Saing to: ', output_file)
+        saver(all_df, output_file)
+    else:
+        ic(all_df)
+    if row_list_filtered_out:
+        df_filtered_out = pd.DataFrame(row_list_filtered_out)
+        ic('Saving filtered out to: ', output_file_filtered_out)
+        saver(df_filtered_out, output_file_filtered_out)

table_processor/core/functions/assign_id.py

@@ -0,0 +1,75 @@
+# -*- coding: utf-8 -*-
+
+import dataclasses
+import json
+import os
+
+from collections import OrderedDict
+from collections import defaultdict
+from typing import Mapping
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+
+# local
+
+from .. constants import (
+    STAGING_FIELD,
+)
+
+from .. config import (
+    AssignIdConfig,
+    Config,
+)
+
+from . search_column_value import search_column_value
+from . set_nested_field_value import set_nested_field_value
+from . set_row_value import set_row_staging_value
+
+from .. types import (
+    AssignIdConfig,
+    IdContextMap,
+    Row,
+)
+
+def assign_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    config: Config,
+):
+    context_columns = []
+    context_values = []
+    if config.context:
+        for context_column in config.context:
+            value, found = search_column_value(row.nested, context_column)
+            if not found:
+                raise KeyError(f'Column not found: {context_column}, existing columns: {row.flat.keys()}')
+            context_columns.append(context_column)
+            context_values.append(value)
+    primary_columns = []
+    primary_values = []
+    for primary_column in config.primary:
+        value, found = search_column_value(row.nested, primary_column)
+        if not found:
+            raise KeyError(f'Column not found: {primary_column}, existing columns: {row.flat.keys()}')
+        primary_columns.append(primary_column)
+        primary_values.append(value)
+    context_key = (
+        tuple(context_columns),
+        tuple(context_values),
+        tuple(primary_columns),
+    )
+    primary_value = tuple(primary_values)
+    id_map = id_context_map[context_key]
+    if primary_value not in id_map.dict_value_to_id:
+        field_id = id_map.max_id + 1
+        id_map.max_id = field_id
+        id_map.dict_value_to_id[primary_value] = field_id
+        id_map.dict_id_to_value[field_id] = primary_value
+    else:
+        field_id = id_map.dict_value_to_id[primary_value]
+    set_row_staging_value(row, config.target, field_id)
+    return row

table_processor/core/functions/flatten_row.py

@@ -0,0 +1,24 @@
+'''
+This module contains the function to flatten a nested dictionary.
+'''
+
+from collections import OrderedDict
+from typing import Mapping
+
+type FlatFieldMap = Mapping[str]
+type FieldMap = Mapping[str, str|FieldMap]
+
+def flatten_row(
+    mapping: FieldMap,
+    parent_key: str = '',
+    new_mapping: FlatFieldMap | None = None,
+) -> FlatFieldMap:
+    if new_mapping is None:
+        new_mapping = OrderedDict()
+    for key, mapped in mapping.items():
+        new_key = f'{parent_key}.{key}' if parent_key else key
+        if isinstance(mapped, Mapping):
+            flatten_row(mapped, new_key, new_mapping)
+        else:
+            new_mapping[new_key] = mapped
+    return new_mapping

table_processor/core/functions/get_nested_field_value.py

@@ -0,0 +1,22 @@
+# Description: Get the value of a field in a dictionary.
+
+from collections import OrderedDict
+
+def get_nested_field_value(
+    data: OrderedDict | list,
+    field: str,
+):
+    if isinstance(data, list):
+        if field.isdigit():
+            index = int(field)
+            if index < len(data):
+                return data[index], True
+            return None, False
+    if isinstance(data, dict):
+        if field in data:
+            return data[field], True
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field in data:
+            return get_nested_field_value(data[field], rest)
+    return None, False

table_processor/core/functions/nest_row.py

@@ -0,0 +1,25 @@
+'''
+This function is used to nest a row. It is used to nest a row that has been unnested.
+'''
+
+import math
+
+from collections import OrderedDict
+from typing import Mapping
+
+from .set_nested_field_value import set_nested_field_value
+
+def nest_row(
+    row: Mapping,
+    remove_nan: bool = True,
+):
+    new_row = OrderedDict()
+    for key, value in row.items():
+        if isinstance(value, OrderedDict):
+            value = nest_row(value)
+        if isinstance(value, float):
+            if math.isnan(value):
+                if remove_nan:
+                    continue
+        set_nested_field_value(new_row, key, value)
+    return new_row

table_processor/core/functions/search_column_value.py

@@ -0,0 +1,26 @@
+'''
+This function is used to search for a column value in a row. It will first search in the '__debug__' field, then in the '__debug__.__original__' field, and finally in the row itself. If the value is found, it will be set in the row and returned.
+'''
+
+from .. constants import (
+    INPUT_FIELD,
+    STAGING_FIELD,
+)
+
+from collections import OrderedDict
+
+from . get_nested_field_value import get_nested_field_value
+
+def search_column_value(
+    row: OrderedDict,
+    column: str,
+):
+    for key in [
+        f'{STAGING_FIELD}.{column}',
+        column,
+        f'{STAGING_FIELD}.{INPUT_FIELD}.{column}',
+    ]:
+        value, found = get_nested_field_value(row, key)
+        if found:
+            return value, key
+    return None, None

table_processor/core/functions/set_flat_field_value.py

@@ -0,0 +1,23 @@
+'''
+Set a value in a flat row dictionary with a nested key.
+'''
+
+from collections import OrderedDict
+from typing import Any
+
+def set_flat_field_value(
+    flat_row: OrderedDict,
+    target: str,
+    value: Any,
+    depth: int = 0,
+):
+    if depth > 10:
+        raise ValueError(
+            'Depth too high'
+        )
+    if isinstance(value, dict):
+        for key in value.keys():
+            set_flat_field_value(flat_row, f'{target}.{key}', value[key], depth + 1)
+    else:
+        flat_row[target] = value
+    return flat_row

table_processor/core/functions/set_nested_field_value.py

@@ -0,0 +1,24 @@
+'''
+Set the value of a field in a nested dictionary.
+'''
+
+from collections import OrderedDict
+from icecream import ic
+
+def set_nested_field_value(
+    data: OrderedDict,
+    field: str,
+    value: any,
+):
+    if isinstance(field, str) and  '.' in field:
+        field, rest = field.split('.', 1)
+        sub_data = data.get(field)
+        if not isinstance(sub_data, dict):
+            data[field] = OrderedDict()
+        set_nested_field_value(data[field], rest, value)
+    else:
+        try:
+            data[field] = value
+        except:
+            ic(data, field, value)
+            raise

table_processor/core/functions/set_row_value.py

@@ -0,0 +1,33 @@
+'''
+Set a value in a row, both in the flat and nested representations.
+'''
+
+from typing import Any
+
+from .. constants import (
+    STAGING_FIELD,
+)
+
+from . set_flat_field_value import set_flat_field_value
+from . set_nested_field_value import set_nested_field_value
+
+from .. types import (
+    Row,
+)
+
+def set_row_value(
+    row: Row,
+    target: str,
+    value: Any,
+):
+    set_flat_field_value(row.flat, target, value)
+    set_nested_field_value(row.nested, target, value)
+    return row
+
+def set_row_staging_value(
+    row: Row,
+    target: str,
+    value: Any,
+):
+    set_row_value(row, f'{STAGING_FIELD}.{target}', value)
+    return row