python-table-processor 0.3.11__py3-none-any.whl → 0.3.12__py3-none-any.whl

{python_table_processor-0.3.11.dist-info → python_table_processor-0.3.12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-processor
-Version: 0.3.11
+Version: 0.3.12
 Summary: A table data processor
 Home-page: https://github.com/akivajp/python-table-processor
 License: MIT

{python_table_processor-0.3.11.dist-info → python_table_processor-0.3.12.dist-info}/RECORD

@@ -1,11 +1,11 @@
-table_processor/__init__.py,sha256=VGVBf0Tvu9kKvvklW4UI-fweoMZobinpwjqSjD7Hm30,54
+table_processor/__init__.py,sha256=C_N1IcjNNCB7P46AT3oYEFKtst2Syoxy6Y823dznVKI,101
 table_processor/cli.py,sha256=uluf_VsBzq3Ldxh3ePGseSrv8--lI7huu-6RjDaHPPg,2170
 table_processor/commands/convert_tables.py,sha256=w4IZftSlrbt-BMPjldULT_616LSiNE3h7GmrFF7K5OM,2276
 table_processor/commands/merge_tables.py,sha256=fIk3e-Qbdj3QtOXetC4ItbJs9NfBHVsraRhdfnpy2RQ,2194
 table_processor/core/actions.py,sha256=3yjSW1CKXPn49BNHx0HbjkU5gp8266s5iun57snXqlc,23815
 table_processor/core/config.py,sha256=Z1s2r3y4oQXdbDCwIY_hYzoq6VRv7Sb3oUlfBCu13Lc,11646
 table_processor/core/constants.py,sha256=w-1TxtpXywb6Qh8M4YWuzcCHUvCjy5zJYxaXC3RQS6A,158
-table_processor/core/convert.py,sha256=yF_ypp0WCdyavhLldP_dQE9aw9Bm5bS8DQgsCQTTT2E,11562
+table_processor/core/convert.py,sha256=uyoHM0KUNztrRUHAeUDEo9aKrddBt3HjGc3veQeP4BI,6581
 table_processor/core/functions/assign_id.py,sha256=dA-wgZA7E01gbx8qGvo1PeMd3aq9PhMdyITzr3BKXIE,4204
 table_processor/core/functions/flatten_row.py,sha256=2l-s4YXgBM58IthpfVzd7L4EGUCekSi7ss5CaM_3MUU,686
 table_processor/core/functions/get_nested_field_value.py,sha256=ZEJi8XtDONoYmhPoqYpVZRGtEWvsfAksV4FXtkMAaq8,946
@@ -14,10 +14,16 @@ table_processor/core/functions/search_column_value.py,sha256=GscwLaMTORHFDhdCdjz
 table_processor/core/functions/set_flat_field_value.py,sha256=7-BgD2yEUGyBaFaIJ18bYv82Qdy0CN8NiC5ZtsAdv44,531
 table_processor/core/functions/set_nested_field_value.py,sha256=351AoyL8-dT9I4eBKYlxUME03ACiMjDukebzmhOBaks,1308
 table_processor/core/functions/set_row_value.py,sha256=wlF_nVGZY74XCzd6pBJJyti62m6y0s1kc5Q6uq9jn8o,641
-table_processor/core/merge.py,sha256=gcMQ5LgBssGBZ0CuSPN0ddslNuLznxwh024kRiGOFLM,5800
+table_processor/core/io/__init__.py,sha256=vk050otP2V9sH8F9tZSGVQjvYXqBPAGikt7QrmXeZgU,167
+table_processor/core/io/io_csv.py,sha256=sjnDO58HerAk2cz21L0IM_Wh6fKTQtu-N_MlNN3Sbxc,986
+table_processor/core/io/io_excel.py,sha256=-Nvx56W7GI845UdEzcSf0SzCJGtdKPHXLZ3F2jam6f8,1306
+table_processor/core/io/io_json.py,sha256=oViNj3jmhF1ZFwzula1QXwRVqgZ8PdfG-TPmcebyNbo,2055
+table_processor/core/io/loader.py,sha256=HtHZIY396aGMNpPgvSURLNteMJTTD1uTdMabsAw1_Fg,710
+table_processor/core/io/saver.py,sha256=W9pxgx30E8IAq3WwxOA-t0VP22DgSCw7i5lFWZPoMqg,627
+table_processor/core/merge.py,sha256=9vk55j7WOP7mbzqXJ5wVxNO7W3c0RM3Vwck8g6TZChM,6019
 table_processor/core/types.py,sha256=76Fn1PQgO03Swe7E_zFLjusbQ6w9sfxzsL08uHS21hE,2919
-python_table_processor-0.3.11.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
-python_table_processor-0.3.11.dist-info/METADATA,sha256=scfDAfEQ1Gm7-BmJgJEC_XeGzjGAP87OmyQV5aADoko,1099
-python_table_processor-0.3.11.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-python_table_processor-0.3.11.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
-python_table_processor-0.3.11.dist-info/RECORD,,
+python_table_processor-0.3.12.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
+python_table_processor-0.3.12.dist-info/METADATA,sha256=ocXqsLZCnCgPN7uFQ_uu51IE54DYEvTH8UrTii8IJF8,1099
+python_table_processor-0.3.12.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+python_table_processor-0.3.12.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
+python_table_processor-0.3.12.dist-info/RECORD,,

table_processor/__init__.py

@@ -1,2 +1,7 @@
-__version__ = "0.3.11"
-__version_tuple__ = (0, 3, 11)
+__version__ = "0.3.12"
+__version_tuple__ = (0, 3, 12)
+
+from . core.io import (
+    load,
+    save,
+)

table_processor/core/convert.py

@@ -30,10 +30,8 @@ from . constants import (
     INPUT_FIELD,
     STAGING_FIELD,
 )
-from . functions.flatten_row import flatten_row
 from . functions.get_nested_field_value import get_nested_field_value
 from . functions.get_nested_field_value import get_nested_field_value
-from . functions.nest_row import nest_row as nest
 from . functions.search_column_value import search_column_value
 from . functions.set_nested_field_value import set_nested_field_value
 from . functions.set_row_value import (
@@ -53,199 +51,12 @@ from . types import (
     GlobalStatus,
 )
 
-dict_loaders: dict[str, callable] = {}
-def register_loader(
-    ext: str,
-):
-    def decorator(loader):
-        dict_loaders[ext] = loader
-        return loader
-    return decorator
-
-def load(
-    input_file: str,
-    **kwargs,
-):
-    ext = os.path.splitext(input_file)[1]
-    if ext not in dict_loaders:
-        raise ValueError(f'Unsupported file type: {ext}')
-    loader = dict_loaders[ext]
-    return loader(input_file, **kwargs)
-
-dict_savers: dict[str, callable] = {}
-def register_saver(
-    ext: str,
-):
-    def decorator(saver):
-        dict_savers[ext] = saver
-        return saver
-    return decorator
-
-def save(
-    df: pd.DataFrame,
-    output_file: str,
-):
-    ext = os.path.splitext(output_file)[1]
-    if ext not in dict_savers:
-        raise ValueError(f'Unsupported file type: {ext}')
-    saver = dict_savers[ext]
-    saver(df, output_file)
-
-@register_loader('.csv')
-def load_csv(
-    input_file: str,
-    **kwargs,
-):
-    skip_header = kwargs.get('skip_header', False)
-    # utf-8
-    #df = pd.read_csv(input_file)
-    # UTF-8 with BOM
-    if skip_header:
-        df = pd.read_csv(
-            input_file,
-            encoding='utf-8-sig',
-            header=None,
-        )
-        #new_column_names = [f'__values__.{i}' for i in df.columns]
-        new_column_names = [f'{i}' for i in df.columns]
-        df = df.rename(columns=dict(
-            zip(df.columns, new_column_names)
-        ))
-    else:
-        df = pd.read_csv(
-            input_file,
-            encoding='utf-8-sig',
-        )
-    return df
-
-@register_loader('.xlsx')
-def load_excel(
-    input_file: str,
-    **kwargs,
-):
-    #df = pd.read_excel(input_file)
-    # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
-    df = pd.read_excel(input_file, dtype=str)
-    # NOTE: 列番号でもアクセスできるようフィールドを追加する
-    df_with_column_number = pd.read_excel(
-        input_file, dtype=str, header=None, skiprows=1
-    )
-    new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
-    df2 = df_with_column_number.rename(columns=dict(
-        zip(df_with_column_number.columns, new_column_names)
-    ))
-    df = pd.concat([df, df2], axis=1)
-    df = df.dropna(axis=0, how='all')
-    df = df.dropna(axis=1, how='all')
-    return df
-
-@register_loader('.json')
-def load_json(
-    input_file: str,
-    **kiwargs,
-):
-    with open(input_file, 'r') as f:
-        data = json.load(f)
-    if not isinstance(data, list):
-        raise ValueError(f'Invalid JSON array data: {input_file}')
-    #ic(data[0])
-    rows = []
-    for row in data:
-        new_row = flatten_row(row)
-        rows.append(new_row)
-    df = pd.DataFrame(rows)
-    return df
-
-@register_saver('.json')
-def save_json(
-    df: pd.DataFrame,
-    output_file: str,
-):
-    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
-    #df.to_json(
-    #    output_file,
-    #    orient='records',
-    #    force_ascii=False,
-    #    indent=2,
-    #    escape_forward_slashes=False,
-    #)
-    #ic(df.iloc[0])
-    data = df.to_dict(orient='records')
-    #ic(data[0])
-    data = [nest(row) for row in data]
-    #ic(data[0])
-    with open(output_file, 'w') as f:
-        json.dump(
-            data,
-            f,
-            indent=2,
-            ensure_ascii=False,
-        )
-
-@register_loader('.jsonl')
-def load_jsonl(
-    input_file: str,
-    **kwargs,
-):
-    rows = []
-    with open(input_file, 'r') as f:
-        for line in f:
-            row = json.loads(line)
-            rows.append(row)
-    df = pd.DataFrame(rows)
-    return df
-
-@register_saver('.jsonl')
-def save_jsonl(
-    df: pd.DataFrame,
-    output_file: str,
-):
-    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
-    #df.to_json(
-    #    output_file,
-    #    orient='records',
-    #    lines=True,
-    #    force_ascii=False,
-    #)
-    with open(output_file, 'w') as f:
-        for index, row in df.iterrows():
-            data = row.to_dict()
-            json.dump(
-                data,
-                f,
-                ensure_ascii=False,
-            )
-            f.write('\n')
-
-@register_saver('.csv')
-def save_csv(
-    df: pd.DataFrame,
-    output_file: str,
-):
-    # utf-8
-    #df.to_csv(output_file, index=False)
-    # UTF-8 with BOM
-    df.to_csv(output_file, index=False, encoding='utf-8-sig')
-
-@register_saver('.xlsx')
-def save_excel(
-    df: pd.DataFrame,
-    output_file: str,
-):
-    # openpyxl
-    df.to_excel(output_file, index=False)
-    # xlsxwriter
-    #writer = pd.ExcelWriter(
-    #    output_file,
-    #    engine='xlsxwriter',
-    #    engine_kwargs={
-    #        'options': {
-    #            'strings_to_urls': False,
-    #        },
-    #    }
-    #)
-    #df.to_excel(writer, index=False)
-    #writer.close()
+from . io import (
+    get_loader,
+    get_saver,
+    load,
+    save,
+)
 
 def assign_array(
     row: OrderedDict,
@@ -317,10 +128,7 @@ def convert(
             action_delimiter=action_delimiter
         )
     if output_file:
-        ext = os.path.splitext(output_file)[1]
-        if ext not in dict_savers:
-            raise ValueError(f'Unsupported file type: {ext}')
-        saver = dict_savers[ext]
+        saver = get_saver(output_file)
     ic(config)
     #return # debug return
     for input_file in input_files:

table_processor/core/io/__init__.py

@@ -0,0 +1,12 @@
+from . loader import (
+    get_loader,
+    load,
+)
+from . saver import (
+    get_saver,
+    save,
+)
+
+from . import io_csv
+from . import io_excel
+from . import io_json

table_processor/core/io/io_csv.py

@@ -0,0 +1,41 @@
+import pandas as pd
+
+from . loader import register_loader
+from . saver import register_saver
+
+@register_loader('.csv')
+def load_csv(
+    input_file: str,
+    **kwargs,
+):
+    skip_header = kwargs.get('skip_header', False)
+    # utf-8
+    #df = pd.read_csv(input_file)
+    # UTF-8 with BOM
+    if skip_header:
+        df = pd.read_csv(
+            input_file,
+            encoding='utf-8-sig',
+            header=None,
+        )
+        #new_column_names = [f'__values__.{i}' for i in df.columns]
+        new_column_names = [f'{i}' for i in df.columns]
+        df = df.rename(columns=dict(
+            zip(df.columns, new_column_names)
+        ))
+    else:
+        df = pd.read_csv(
+            input_file,
+            encoding='utf-8-sig',
+        )
+    return df
+
+@register_saver('.csv')
+def save_csv(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # utf-8
+    #df.to_csv(output_file, index=False)
+    # UTF-8 with BOM
+    df.to_csv(output_file, index=False, encoding='utf-8-sig')

table_processor/core/io/io_excel.py

@@ -0,0 +1,45 @@
+import pandas as pd
+
+from . loader import register_loader
+from . saver import register_saver
+
+@register_loader('.xlsx')
+def load_excel(
+    input_file: str,
+    **kwargs,
+):
+    #df = pd.read_excel(input_file)
+    # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
+    df = pd.read_excel(input_file, dtype=str)
+    # NOTE: 列番号でもアクセスできるようフィールドを追加する
+    df_with_column_number = pd.read_excel(
+        input_file, dtype=str, header=None, skiprows=1
+    )
+    new_column_names = [f'__values__.{i}' for i in df_with_column_number.columns]
+    df2 = df_with_column_number.rename(columns=dict(
+        zip(df_with_column_number.columns, new_column_names)
+    ))
+    df = pd.concat([df, df2], axis=1)
+    df = df.dropna(axis=0, how='all')
+    df = df.dropna(axis=1, how='all')
+    return df
+
+@register_saver('.xlsx')
+def save_excel(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # openpyxl
+    df.to_excel(output_file, index=False)
+    # xlsxwriter
+    #writer = pd.ExcelWriter(
+    #    output_file,
+    #    engine='xlsxwriter',
+    #    engine_kwargs={
+    #        'options': {
+    #            'strings_to_urls': False,
+    #        },
+    #    }
+    #)
+    #df.to_excel(writer, index=False)
+    #writer.close()

table_processor/core/io/io_json.py

@@ -0,0 +1,86 @@
+import json
+import pandas as pd
+
+from . loader import register_loader
+from . saver import register_saver
+
+from .. functions.flatten_row import flatten_row
+from .. functions.nest_row import nest_row
+
+@register_loader('.json')
+def load_json(
+    input_file: str,
+    **kiwargs,
+):
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f'Invalid JSON array data: {input_file}')
+    #ic(data[0])
+    rows = []
+    for row in data:
+        new_row = flatten_row(row)
+        rows.append(new_row)
+    df = pd.DataFrame(rows)
+    return df
+
+@register_saver('.json')
+def save_json(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    force_ascii=False,
+    #    indent=2,
+    #    escape_forward_slashes=False,
+    #)
+    #ic(df.iloc[0])
+    data = df.to_dict(orient='records')
+    #ic(data[0])
+    data = [nest_row(row) for row in data]
+    #ic(data[0])
+    with open(output_file, 'w') as f:
+        json.dump(
+            data,
+            f,
+            indent=2,
+            ensure_ascii=False,
+        )
+
+@register_loader('.jsonl')
+def load_jsonl(
+    input_file: str,
+    **kwargs,
+):
+    rows = []
+    with open(input_file, 'r') as f:
+        for line in f:
+            row = json.loads(line)
+            rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
+@register_saver('.jsonl')
+def save_jsonl(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    # NOTE: この方法だとスラッシュがすべてエスケープされてしまった
+    #df.to_json(
+    #    output_file,
+    #    orient='records',
+    #    lines=True,
+    #    force_ascii=False,
+    #)
+    with open(output_file, 'w') as f:
+        for index, row in df.iterrows():
+            data = row.to_dict()
+            json.dump(
+                data,
+                f,
+                ensure_ascii=False,
+            )
+            f.write('\n')

table_processor/core/io/loader.py

@@ -0,0 +1,33 @@
+import os
+from typing import Any, Protocol
+
+import pandas as pd
+
+class LoaderType(Protocol):
+    def __call__(self, input_file: str, **kwargs: Any) -> pd.DataFrame:
+         ...
+
+dict_loaders: dict[str, LoaderType] = {}
+def register_loader(
+    ext: str,
+):
+    def decorator(loader):
+        dict_loaders[ext] = loader
+        return loader
+    return decorator
+
+def get_loader(
+    input_file: str,
+):
+    ext = os.path.splitext(input_file)[1]
+    if ext not in dict_loaders:
+        raise ValueError(f'Unsupported file type: {ext}')
+    loader = dict_loaders[ext]
+    return loader
+
+def load(
+    input_file: str,
+    **kwargs,
+):
+    loader = get_loader(input_file)
+    return loader(input_file, **kwargs)

table_processor/core/io/saver.py

@@ -0,0 +1,31 @@
+import os.path
+import pandas as pd
+
+from typing import Callable
+
+type Saver = Callable[[pd.DataFrame, str], None]
+
+dict_savers: dict[str, Saver] = {}
+def register_saver(
+    ext: str,
+):
+    def decorator(saver):
+        dict_savers[ext] = saver
+        return saver
+    return decorator
+
+def get_saver(
+    output_file: str,
+):
+    ext = os.path.splitext(output_file)[1]
+    if ext not in dict_savers:
+        raise ValueError(f'Unsupported file type: {ext}')
+    saver = dict_savers[ext]
+    return saver
+
+def save(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    saver = get_saver(output_file)
+    saver(df, output_file)

table_processor/core/merge.py

@@ -34,7 +34,9 @@ from . actions import (
     prepare_row,
 )
 
-from . convert import (
+from . io import (
+    get_loader,
+    get_saver,
     load,
     save,
 )
@@ -75,6 +77,13 @@ def merge(
     all_modified_rows = []
     list_ignored_keys = []
     num_modified = 0
+    for output_path in [
+        output_base_data_file,
+        output_modified_data_file,
+        output_remaining_data_file,
+    ]:
+        if output_path:
+            get_saver(output_path)
     for previous_file in previous_files:
         if not os.path.exists(previous_file):
             raise FileNotFoundError(f'File not found: {previous_file}')