python-table-converter 0.1.0__tar.gz

python_table_converter-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Akiva Miura
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

python_table_converter-0.1.0/PKG-INFO

@@ -0,0 +1,24 @@
+Metadata-Version: 2.1
+Name: python-table-converter
+Version: 0.1.0
+Summary: A table data converter
+License: MIT
+Author: Akiva Miura
+Author-email: akiva.miura@gmail.com
+Requires-Python: >=3.10,<4.0
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.10
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Requires-Dist: icecream (>=2.1.3,<3.0.0)
+Requires-Dist: logzero (>=1.7.0,<2.0.0)
+Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
+Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: tqdm (>=4.67.0,<5.0.0)
+Description-Content-Type: text/markdown
+
+# Table Data Converter
+
+This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.
+

python_table_converter-0.1.0/README.md

@@ -0,0 +1,3 @@
+# Table Data Converter
+
+This is a python-based tool that converts tables from one format to another. It can convert tables from CSV, TSV, Excel, JSON, and JSON Lines to any of these formats.

python_table_converter-0.1.0/pyproject.toml

@@ -0,0 +1,32 @@
+[tool.poetry]
+#name = "table-converter"
+name = "python-table-converter"
+version = "0.1.0"
+description = "A table data converter"
+authors = ["Akiva Miura <akiva.miura@gmail.com>"]
+license = "MIT"
+readme = "README.md"
+packages = [
+    { include = "table_converter", from = "." },
+]
+
+#[tool.poetry-dynamic-versioning]
+#enable = true
+#style = "pep440"
+
+[tool.poetry.scripts]
+table-converter = "table_converter.cli:main"
+conver-tables = "table_converter.cli:command_convert_tables"
+
+[tool.poetry.dependencies]
+python = "^3.10"
+icecream = "^2.1.3"
+logzero = "^1.7.0"
+tqdm = "^4.67.0"
+pandas = "^2.2.3"
+openpyxl = "^3.1.5"
+
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"

python_table_converter-0.1.0/table_converter/cli.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import argparse
+import sys
+
+from icecream import ic
+
+def parse_and_run(
+    parser: argparse.ArgumentParser,
+):
+    ic.disable()
+    ic()
+    args = parser.parse_args()
+    ic(args)
+    if args.handler:
+        args.handler(args)
+    else:
+        parser.print_help()
+        sys.exit(1)
+
+def command_convert_tables(
+    parser: argparse.ArgumentParser|None = None,
+):
+    if parser is None:
+        command_parser = argparse.ArgumentParser(
+            description='Convert a table to a different format.'
+        )
+    else:
+        command_parser = parser
+    from . commands.convert_tables import setup_parser
+    setup_parser(command_parser)
+    if parser is None:
+        parse_and_run(parser)
+
+def main():
+    parser = argparse.ArgumentParser(description='Table Data Converter')
+    parser.set_defaults(handler=None)
+    subparsers = parser.add_subparsers(dest='command')
+
+    parser_convert_tables = subparsers.add_parser(
+        'convert',
+        help='Convert a table to a different format.'
+    )
+    command_convert_tables(parser_convert_tables)
+
+    parse_and_run(parser)
+
+if __name__ == '__main__':
+    main()

python_table_converter-0.1.0/table_converter/commands/convert_tables.py

@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from icecream import ic
+
+from .. core.convert import convert
+
+def run(
+    args: argparse.Namespace,
+):
+    convert(
+        input_files = args.input_files,
+        output_file = args.output_file,
+        assign_constants = args.assign_constants,
+        assign_formats = args.assign_formats,
+        pickup_columns= args.pickup_columns,
+        fields_to_split_by_newline = args.split_by_newline,
+        fields_to_assign_ids = args.assign_ids,
+        output_debug = args.output_debug,
+    )
+
+def setup_parser(
+    parser: argparse.ArgumentParser,
+):
+    parser.add_argument(
+        'input_files',
+        metavar='INPUT_FILE',
+        nargs='+',
+        help='Path to the input file.'
+    )
+    parser.add_argument(
+        '-o', '--output-file',
+        metavar='OUTPUT_FILE',
+        required=True,
+        help='Path to the output file.'
+    )
+    parser.add_argument(
+        '--pickup-columns',
+        type=str,
+        help='Pickup column map',
+    )
+    parser.add_argument(
+        '--split-by-newline',
+        type=str,
+        help='Fields to split by newline',
+    )
+    parser.add_argument(
+        '--assign-ids',
+        type=str,
+        help='Field to assign ids',
+    )
+    parser.add_argument(
+        '--assign-constants',
+        type=str,
+        help='Field to assign constants',
+    )
+    parser.add_argument(
+        '--assign-formats',
+        type=str,
+        help='Field to assign formats',
+    )
+    parser.add_argument(
+        '--output-debug',
+        action='store_true',
+        help='Output debug information',
+    )
+    parser.set_defaults(handler=run)

python_table_converter-0.1.0/table_converter/core/convert.py

@@ -0,0 +1,345 @@
+# -*- coding: utf-8 -*-
+
+import json
+import os
+
+from collections import OrderedDict
+
+import numpy as np
+import pandas as pd
+from icecream import ic
+
+dict_loaders: dict[str, callable] = {}
+def register_loader(
+    ext: str,
+):
+    def decorator(loader):
+        dict_loaders[ext] = loader
+        return loader
+    return decorator
+
+dict_savers: dict[str, callable] = {}
+def register_saver(
+    ext: str,
+):
+    def decorator(saver):
+        dict_savers[ext] = saver
+        return saver
+    return decorator
+
+@register_loader('.xlsx')
+def load_excel(
+    input_file: str,
+):
+    df = pd.read_excel(input_file)
+    return df
+
+@register_saver('.json')
+def save_json(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    force_ascii=False,
+    #    indent=2,
+    #    escape_forward_slashes=False,
+    #)
+    #ic(df.iloc[0])
+    data = df.to_dict(orient='records')
+    with open(output_file, 'w') as f:
+        json.dump(
+            data,
+            f,
+            indent=2,
+            ensure_ascii=False,
+        )
+
+@register_saver('.jsonl')
+def save_jsonl(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    lines=True,
+    #    force_ascii=False,
+    #)
+    with open(output_file, 'w') as f:
+        for index, row in df.iterrows():
+            data = row.to_dict()
+            json.dump(
+                data,
+                f,
+                ensure_ascii=False,
+            )
+            f.write('\n')
+
+def set_field_value(
+    data: OrderedDict,
+    field: str,
+    value: any,
+):
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field not in data:
+            data[field] = OrderedDict()
+        set_field_value(data[field], rest, value)
+    else:
+        data[field] = value
+
+def get_field_value(
+    data: OrderedDict,
+    field: str,
+):
+    if field in data:
+        return data[field], True
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field in data:
+            return get_field_value(data[field], rest)
+    return None, False
+
+def search_column_value(
+    row: OrderedDict,
+    column: str,
+):
+    if '__debug__' in row:
+        value, found = get_field_value(row['__debug__'], column)
+        if found:
+            return value, True
+    value, found = get_field_value(row['__debug__'], column)
+    original, found = get_field_value(row, '__debug__.__original__')
+    if found:
+        value, found = get_field_value(original, column)
+        if found:
+            return value, True
+    value, found = get_field_value(row, column)
+    if found:
+        set_field_value(row, column, value)
+        return value, True
+    return None, False
+
+def map_constants(
+    row: OrderedDict,
+    dict_constants: OrderedDict,
+):
+    new_row = OrderedDict(row)
+    for column in dict_constants.keys():
+        #set_field_value(new_row, column, dict_constants[column])
+        set_field_value(new_row, f'__debug__.{column}', dict_constants[column])
+    return new_row
+
+def map_formats(
+    row: OrderedDict,
+    dict_formats: OrderedDict,
+):
+    new_row = OrderedDict(row)
+    for column in dict_formats.keys():
+        template = dict_formats[column]
+        try:
+            params = {}
+            params.update(row['__debug__'])
+            params.update(row)
+            formatted = template.format(**params)
+        except KeyError as e:
+            ic(e)
+            formatted = template
+            raise e
+        set_field_value(new_row, f'__debug__.{column}', formatted)
+    return new_row
+
+def remap_columns(
+    row: OrderedDict,
+    dict_remap: OrderedDict,
+):
+    new_row = OrderedDict()
+    for column in dict_remap.keys():
+        value, found = search_column_value(row, dict_remap[column])
+        if found:
+            set_field_value(new_row, column, value)
+    for column in row.keys():
+        if column == '__debug__':
+            # NOTE: Ignore debug fields
+            set_field_value(new_row, column, row[column])
+    return new_row
+
+def apply_fields_split_by_newline(
+    row: OrderedDict,
+    dict_fields: OrderedDict,
+):
+    new_row = OrderedDict(row)
+    for column in dict_fields:
+        value, found = search_column_value(row, dict_fields[column])
+        #ic(value, found)
+        if found:
+            if isinstance(value, str):
+                new_value = value.split('\n')
+                set_field_value(new_row, f'__debug__.{column}', new_value)
+            else:
+                set_field_value(new_row, f'__debug__.{column}', value)
+    return new_row
+
+def create_id_stat_node():
+    return {
+        'max_id': 0,
+        'dict_value_to_id': {},
+        'dict_id_to_node': {},
+    }
+
+def assign_id_in_node(
+    row: OrderedDict,
+    column: str,
+    dict_assignment: OrderedDict,
+    id_stat_node: dict,
+):
+    value, found = search_column_value(row, dict_assignment[column])
+    if not found:
+        raise KeyError(f'Column not found: {column}, existing columns: {row.keys()}')
+    if value not in id_stat_node['dict_value_to_id']:
+        field_id = id_stat_node['max_id'] + 1
+        id_stat_node['max_id'] = field_id
+        id_stat_node['dict_value_to_id'][value] = field_id
+        node = create_id_stat_node()
+        id_stat_node['dict_id_to_node'][field_id] = node
+    else:
+        field_id = id_stat_node['dict_value_to_id'][value]
+        node = id_stat_node['dict_id_to_node'][field_id]
+    set_field_value(row, f'__debug__.{column}', field_id)
+    set_field_value(row, f'__debug__.__ids__.{column}', field_id)
+    return node
+
+def assign_id(
+    row: OrderedDict,
+    dict_assignment: OrderedDict,
+    root_id_stat_node: dict,
+):
+    new_row = OrderedDict(row)
+    node = root_id_stat_node
+    for column in dict_assignment:
+        node = assign_id_in_node(new_row, column, dict_assignment, node)
+    return new_row
+
+def convert(
+    input_files: list[str],
+    output_file: str | None = None,
+    assign_constants: str | None = None,
+    assign_formats: str | None = None,
+    pickup_columns: str | None = None,
+    fields_to_split_by_newline: str | None = None,
+    fields_to_assign_ids: str | None = None,
+    output_debug: bool = False,
+):
+    ic()
+    ic(input_files)
+    df_list = []
+    dict_constants: OrderedDict | None = None
+    dict_columns: OrderedDict | None = None
+    dict_formats: OrderedDict | None = None
+    #list_fields_to_split_by_newline = None
+    dict_split_by_newline: OrderedDict | None = None
+    dict_assign_ids= None
+    root_id_stat = create_id_stat_node()
+    if assign_constants:
+        dict_constants = OrderedDict()
+        fields = assign_constants.split(',')
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                dict_constants[dst] = src
+            else:
+                raise ValueError(f'Invalid constant assignment: {field}')
+    if assign_formats:
+        dict_formats = OrderedDict()
+        fields = assign_formats.split(',')
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                dict_formats[dst] = src
+            else:
+                raise ValueError(f'Invalid template assignment: {field}')
+    if pickup_columns:
+        dict_columns = OrderedDict()
+        fields = pickup_columns.split(',')
+        for field in fields:
+            if '=' in field:
+                dst, value = field.split('=')
+                dict_columns[dst] = value
+            else:
+                dict_columns[field] = field
+    if fields_to_split_by_newline:
+        #list_fields_to_split_by_newline = fields_to_split_by_newline.split(',')
+        dict_split_by_newline = OrderedDict()
+        fields = fields_to_split_by_newline.split(',')
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                dict_split_by_newline[dst] = src
+            else:
+                raise ValueError(f'Invalid split by newline: {field}')
+    if fields_to_assign_ids:
+        dict_assign_ids = OrderedDict()
+        fields = fields_to_assign_ids.split(',')
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                dict_assign_ids[dst] = src
+            else:
+                raise ValueError(f'Invalid id assignment: {field}')
+    if output_file:
+        ext = os.path.splitext(output_file)[1]
+        if ext not in dict_savers:
+            raise ValueError(f'Unsupported file type: {ext}')
+        saver = dict_savers[ext]
+    for input_file in input_files:
+        ic(input_file)
+        if not os.path.exists(input_file):
+            raise FileNotFoundError(f'File not found: {input_file}')
+        ext = os.path.splitext(input_file)[1]
+        ic(ext)
+        if ext not in dict_loaders:
+            raise ValueError(f'Unsupported file type: {ext}')
+        df = dict_loaders[ext](input_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        ic(len(df))
+        ic(df.columns)
+        ic(df.iloc[0])
+        new_rows = []
+        for index, row in df.iterrows():
+            orig = OrderedDict(row)
+            new_row = OrderedDict(row)
+            set_field_value(new_row, '__debug__.__original__', orig)
+            set_field_value(new_row, '__debug__.__file__', input_file)
+            if dict_constants:
+                new_row = map_constants(new_row, dict_constants)
+            if dict_columns:
+                new_row = remap_columns(new_row, dict_columns)
+            #if list_fields_to_split_by_newline:
+            #    new_row = apply_fields_split_by_newline(new_row, list_fields_to_split_by_newline)
+            if dict_split_by_newline:
+                new_row = apply_fields_split_by_newline(new_row, dict_split_by_newline)
+            if dict_assign_ids:
+                new_row = assign_id(new_row, dict_assign_ids, root_id_stat)
+            if dict_formats:
+                new_row = map_formats(new_row, dict_formats)
+            if dict_columns:
+                new_row = remap_columns(new_row, dict_columns)
+            if not output_debug:
+                new_row.pop('__debug__', None)
+            new_rows.append(new_row)
+        new_df = pd.DataFrame(new_rows)
+        df_list.append(new_df)
+    all_df = pd.concat(df_list)
+    #ic(all_df)
+    ic(len(all_df))
+    ic(all_df.columns)
+    ic(all_df.iloc[0])
+    if output_file:
+        ic('Saing to: ', output_file)
+        saver(all_df, output_file)