python-table-processor 0.2.29__py3-none-any.whl → 0.3.1__py3-none-any.whl

{python_table_processor-0.2.29.dist-info → python_table_processor-0.3.1.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-processor
-Version: 0.2.29
+Version: 0.3.1
 Summary: A table data processor
 Home-page: https://github.com/akivajp/python-table-processor
 License: MIT

python_table_processor-0.3.1.dist-info/RECORD

@@ -0,0 +1,23 @@
+table_processor/__init__.py,sha256=7U6Yq5pgarhomICnrlEYAI32ycMhvYb4mYoUd4axAyU,52
+table_processor/cli.py,sha256=uluf_VsBzq3Ldxh3ePGseSrv8--lI7huu-6RjDaHPPg,2170
+table_processor/commands/convert_tables.py,sha256=j25KL0zRl5ikrBk73jAkKn4rPmDq7SecChZxecDpILQ,2123
+table_processor/commands/merge_tables.py,sha256=Uu-hjfPh69Of0_OZrY8sY0oscpLSdK4jwJ_t1qgfPMg,1485
+table_processor/core/actions.py,sha256=rXdXXo8AXUaLnrzDp8vRduU0XxIOD7PQdkjHiJzOzq8,19702
+table_processor/core/config.py,sha256=irJxJBhBQNz5cp9XlMjVpZ47hxclG6pkTphK-SCrZQ0,11727
+table_processor/core/constants.py,sha256=w-1TxtpXywb6Qh8M4YWuzcCHUvCjy5zJYxaXC3RQS6A,158
+table_processor/core/convert.py,sha256=2R9zf0SzRZEpNEd51eym10mJ3kb5vLSnsY81z4dDXCA,11566
+table_processor/core/functions/assign_id.py,sha256=dA-wgZA7E01gbx8qGvo1PeMd3aq9PhMdyITzr3BKXIE,4204
+table_processor/core/functions/flatten_row.py,sha256=2l-s4YXgBM58IthpfVzd7L4EGUCekSi7ss5CaM_3MUU,686
+table_processor/core/functions/get_nested_field_value.py,sha256=vF4pH6nxR3-2bDfAvrMfHcPFNiS4MnGkXL8FdcScdIw,685
+table_processor/core/functions/nest_row.py,sha256=b5AKfxE39g-NOLranQRy0dfTZq-27Or8ZlgCB8N5kFs,640
+table_processor/core/functions/search_column_value.py,sha256=GscwLaMTORHFDhdCdjzdoy46UGyFHrGwDQA2b_tZhsQ,740
+table_processor/core/functions/set_flat_field_value.py,sha256=7-BgD2yEUGyBaFaIJ18bYv82Qdy0CN8NiC5ZtsAdv44,531
+table_processor/core/functions/set_nested_field_value.py,sha256=csO0v_nW94q-M9bNL91xR-TLHPXDDxnLK8gN-0LLd6k,595
+table_processor/core/functions/set_row_value.py,sha256=wlF_nVGZY74XCzd6pBJJyti62m6y0s1kc5Q6uq9jn8o,641
+table_processor/core/merge.py,sha256=Yc5OV9n1xV9QtgY4srl6Susd5cOKbuYseEtYuBampDw,4672
+table_processor/core/types.py,sha256=vqm7Jyktrcd3qvfJspu8421MwsXI-jxnCeXAbmpTDHw,2498
+python_table_processor-0.3.1.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
+python_table_processor-0.3.1.dist-info/METADATA,sha256=nAAAO4VF74FJv3Qd-Jd1lNRTQVvEOKDcZWu-13C04_g,1098
+python_table_processor-0.3.1.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+python_table_processor-0.3.1.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
+python_table_processor-0.3.1.dist-info/RECORD,,

table_processor/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.29"
-__version_tuple__ = (0, 2, 29)
+__version__ = "0.3.1"
+__version_tuple__ = (0, 3, 1)

table_processor/cli.py

@@ -34,7 +34,21 @@ def command_convert_tables(
         )
     else:
         command_parser = parser
-    from table_converter.commands.convert_tables import setup_parser
+    from table_processor.commands.convert_tables import setup_parser
+    setup_parser(command_parser)
+    if parser is None:
+        parse_and_run(command_parser)
+
+def command_merge_tables(
+    parser: argparse.ArgumentParser|None = None,
+):
+    if parser is None:
+        command_parser = argparse.ArgumentParser(
+            description='Merge tables.'
+        )
+    else:
+        command_parser = parser
+    from table_processor.commands.merge_tables import setup_parser
     setup_parser(command_parser)
     if parser is None:
         parse_and_run(command_parser)
@@ -60,6 +74,13 @@ def main():
     setup_common_args(parser_convert_tables)
     command_convert_tables(parser_convert_tables)
 
+    parser_merge_tables = subparsers.add_parser(
+        'merge',
+        help='Merge tables.'
+    )
+    setup_common_args(parser_merge_tables)
+    command_merge_tables(parser_merge_tables)
+
     parse_and_run(parser)
 
 if __name__ == '__main__':

table_processor/commands/merge_tables.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from icecream import ic
+
+from .. core.merge import merge
+
+def run(
+    args: argparse.Namespace,
+):
+    merge(
+        previous_files=args.previous_files,
+        modification_files=args.modification_files,
+        keys=args.keys,
+        allow_duplicate_keys=args.allow_duplicate_keys,
+        ignore_not_found=args.ignore_not_found,
+        output_file=args.output_file,
+    )
+
+def setup_parser(
+    parser: argparse.ArgumentParser,
+):
+    parser.add_argument(
+        '--previous-files', '--previous-file', '--previous', '--old', '-P',
+        nargs='+',
+        required=True,
+        help='Previous files to merge',
+    )
+    parser.add_argument(
+        '--modification-files', '--modification', '--modify', '--modified', '--new', '-M',
+        nargs='+',
+        required=True,
+        help='Modification files to merge',
+    )
+    parser.add_argument(
+        '--keys', '-K',
+        nargs='+',
+        required=True,
+        help='Primary keys',
+    )
+    parser.add_argument(
+        '--allow-duplicate-keys', '--allow-duplicate',
+        action='store_true',
+        help='Allow duplicate keys',
+    )
+    parser.add_argument(
+        '--ignore-not-found',
+        action='store_true',
+        help='Ignore not found',
+    )
+    parser.add_argument(
+        '--output-file', '--output', '-o',
+        metavar='OUTPUT_FILE',
+        required=False,
+        help='Path to the output file.',
+    )
+    parser.set_defaults(handler=run)

table_processor/core/actions.py

@@ -34,6 +34,7 @@ from . types import (
     OmitConfig,
     ParseConfig,
     PickConfig,
+    PushConfig,
     SplitConfig,
     Row,
 )
@@ -131,10 +132,12 @@ def setup_actions_with_args(
                 context = options.get('context', None)
                 if context:
                     context = context.split(',')
+                reverse = options.get('reverse', False)
                 config.actions.append(AssignIdConfig(
                     target = target,
                     primary = [source],
                     context = context,
+                    reverse = reverse,
                 ))
                 continue
             if action_name == 'filter-empty':
@@ -187,6 +190,14 @@ def setup_actions_with_args(
                     required = required,
                 ))
                 continue
+            if action_name == 'push':
+                condition = options.get('condition', None)
+                config.actions.append(PushConfig(
+                    target = target,
+                    source = source,
+                    condition = condition,
+                ))
+                continue
             if action_name == 'split':
                 delimiter = options.get('delimiter', None)
                 if delimiter == '\\n':
@@ -302,6 +313,8 @@ def do_action(
         return parse(row, action)
     if isinstance(action, OmitConfig):
         return omit_field(row, action)
+    if isinstance(action, PushConfig):
+        return push_field(row, action)
     if isinstance(action, SplitConfig):
         return split_field(row, action)
     raise ValueError(
@@ -488,7 +501,8 @@ def assign_format(
             params[key] = f'__{key}__undefined__'
         except:
             #ic(params)
-            ic(params.keys())
+            #ic(params.keys())
+            ic(row.flat)
             raise
     set_row_staging_value(row, config.target, formatted)
     return row
@@ -599,3 +613,25 @@ def parse(
             parsed = value
         set_row_staging_value(row, config.target, parsed)
     return row
+
+def push_field(
+    row: Row,
+    config: PushConfig,
+):
+    source_value, found = search_column_value(row.nested, config.source)
+    do_append = False
+    if config.condition is None:
+        do_append = True
+    else:
+        condition_value, found = search_column_value(row.nested, config.condition)
+        if condition_value:
+            do_append = True
+    if do_append:
+        target_value, found = search_column_value(row.nested, config.target)
+        if found:
+            array = target_value
+        else:
+            array = []
+            set_row_staging_value(row, config.target, array)
+        array.append(source_value)
+    return row

table_processor/core/config.py

@@ -31,17 +31,10 @@ class AssignArrayConfig:
     field: str
     optional: bool = True
 
-@dataclasses.dataclass
-class PushConfig:
-    target: str
-    source: str
-    condition: str | None = None
-
 @dataclasses.dataclass
 class ProcessConfig:
     assign_array: Mapping[str, list[AssignArrayConfig]] = dataclasses.field(default_factory=OrderedDict)
     assign_length: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    push: list[PushConfig] = dataclasses.field(default_factory=list)
 
     def __setitem__(self, key, value):
         setattr(self, key, value)

table_processor/core/convert.py

@@ -20,7 +20,6 @@ import pandas as pd
 
 from . config import (
     AssignArrayConfig,
-    PushConfig,
     setup_config,
     setup_pick_with_args,
 )
@@ -63,6 +62,15 @@ def register_loader(
         return loader
     return decorator
 
+def load(
+    input_file: str,
+):
+    ext = os.path.splitext(input_file)[1]
+    if ext not in dict_loaders:
+        raise ValueError(f'Unsupported file type: {ext}')
+    loader = dict_loaders[ext]
+    return loader(input_file)
+
 dict_savers: dict[str, callable] = {}
 def register_saver(
     ext: str,
@@ -72,6 +80,16 @@ def register_saver(
         return saver
     return decorator
 
+def save(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    ext = os.path.splitext(output_file)[1]
+    if ext not in dict_savers:
+        raise ValueError(f'Unsupported file type: {ext}')
+    saver = dict_savers[ext]
+    saver(df, output_file)
+
 @register_loader('.csv')
 def load_csv(
     input_file: str,
@@ -245,28 +263,6 @@ def search_column_value_from_nested(
     return None, False
 
 
-def push_fields(
-    row: OrderedDict,
-    list_config: list[PushConfig],
-):
-    nested_row = nest(row)
-    for config in list_config:
-        target_value, found = search_column_value_from_nested(nested_row, config.target)
-        if found:
-            array = target_value
-        else:
-            array = []
-            #set_field_value(nested_row, f'{STAGING_FIELD}.{config.target}', array)
-            set_row_staging_value(nested_row, config.target, array)
-        source_value, found = search_column_value_from_nested(nested_row, config.source)
-        if config.condition is None:
-            array.append(source_value)
-            continue
-        condition_value, found = search_column_value_from_nested(nested_row, config.condition)
-        if condition_value:
-            array.append(source_value)
-    return flatten_row(nested_row)
-
 def assign_length(
     row: OrderedDict,
     dict_fields: OrderedDict,
@@ -352,8 +348,6 @@ def convert(
                 set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
             if config.process.assign_array:
                 row.flat= assign_array(row.flat, config.process.assign_array)
-            if config.process.push:
-                row.flat = push_fields(row.flat, config.process.push)
             if config.process.assign_length:
                 row.flat = assign_length(row.flat, config.process.assign_length)
             if config.actions:

table_processor/core/functions/assign_id.py

@@ -35,15 +35,17 @@ from .. types import (
     Row,
 )
 
-def assign_id(
-    id_context_map: IdContextMap,
+def get_key_value(
     row: Row,
-    config: Config,
+    primary: list[str],
+    context: list[str],
 ):
+    if not primary:
+        raise ValueError('Primary columns must be specified')
     context_columns = []
     context_values = []
-    if config.context:
-        for context_column in config.context:
+    if context:
+        for context_column in context:
             value, found = search_column_value(row.nested, context_column)
             if not found:
                 raise KeyError(f'Column not found: {context_column}, existing columns: {row.flat.keys()}')
@@ -51,7 +53,7 @@ def assign_id(
             context_values.append(value)
     primary_columns = []
     primary_values = []
-    for primary_column in config.primary:
+    for primary_column in primary:
         value, found = search_column_value(row.nested, primary_column)
         if not found:
             raise KeyError(f'Column not found: {primary_column}, existing columns: {row.flat.keys()}')
@@ -63,13 +65,89 @@ def assign_id(
         tuple(primary_columns),
     )
     primary_value = tuple(primary_values)
+    return context_key, primary_value
+
+def get_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    primary: list[str],
+    context: list[str],
+):
+    context_key, primary_value = get_key_value(
+        row=row,
+        primary=primary,
+        context=context,
+    )
     id_map = id_context_map[context_key]
+    #ic(primary_value, context_key)
+    #ic(primary_value in id_map.dict_value_to_id)
     if primary_value not in id_map.dict_value_to_id:
         field_id = id_map.max_id + 1
         id_map.max_id = field_id
         id_map.dict_value_to_id[primary_value] = field_id
         id_map.dict_id_to_value[field_id] = primary_value
+        id_exists = False
     else:
         field_id = id_map.dict_value_to_id[primary_value]
+        id_exists = True
+    return field_id, id_exists
+
+
+def set_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    primary: list[str],
+    context: list[str],
+    id_value: int,
+):
+    context_key, primary_value = get_key_value(
+        row=row,
+        primary=primary,
+        context=context,
+    )
+    id_map = id_context_map[context_key]
+    if id_value in id_map.dict_id_to_value:
+        #raise ValueError(f'ID already exists: {id_value}')
+        field_id = id_map.dict_id_to_value[id_value]
+        if field_id != primary_value:
+            raise ValueError(f'ID already exists: {id_value} for {field_id}')
+    id_map.dict_value_to_id[primary_value] = id_value
+    id_map.dict_id_to_value[id_value] = primary_value
+    id_map.max_id = max(id_map.max_id, id_value)
+    return
+
+
+def assign_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    config: Config,
+):
+    if config.reverse:
+        #ic(row)
+        #ic(type(row))
+        #ic(config)
+        value, found = search_column_value(row.nested, config.target)
+        #ic(row, value, found)
+        if type(value) is str:
+            if value.isdigit():
+                value = int(value)
+        if type(value) is int:
+            field_id = value
+            set_id(
+                id_context_map=id_context_map,
+                row=row,
+                primary=config.primary,
+                context=config.context,
+                id_value=field_id,
+            )
+            #return field_id, True
+            return row
+    field_id, id_exists = get_id(
+        id_context_map=id_context_map,
+        row=row,
+        primary=config.primary,
+        context=config.context,
+    )
+    #ic(config, field_id, id_exists)
     set_row_staging_value(row, config.target, field_id)
     return row

table_processor/core/functions/get_nested_field_value.py

@@ -2,11 +2,14 @@
 
 from collections import OrderedDict
 
+from icecream import ic
+
 def get_nested_field_value(
     data: OrderedDict | list,
     field: str,
 ):
     if isinstance(data, list):
+        #ic(data, field)
         if field.isdigit():
             index = int(field)
             if index < len(data):

table_processor/core/merge.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+
+import json
+import math
+import os
+
+from collections import OrderedDict
+
+from typing import (
+    Mapping,
+)
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+from tqdm.auto import tqdm
+
+# local
+
+from . functions.flatten_row import flatten_row
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.nest_row import nest_row as nest
+from . functions.search_column_value import search_column_value
+from . functions.set_nested_field_value import set_nested_field_value
+from . functions.set_row_value import (
+    set_row_value,
+    set_row_staging_value,
+)
+
+from . actions import (
+    prepare_row,
+)
+
+from . convert import (
+    load,
+    save,
+)
+
+def merge(
+    previous_files: list[str],
+    modification_files: list[str],
+    keys: list[str],
+    allow_duplicate_keys: bool = False,
+    ignore_not_found: bool = False,
+    output_file: str | None = None,
+):
+    ic.enable()
+    ic()
+    ic(previous_files)
+    ic(modification_files)
+    ic(keys)
+    dict_key_to_row = {}
+    all_rows = []
+    list_ignored_keys = []
+    num_modified = 0
+    for previous_file in previous_files:
+        if not os.path.exists(previous_file):
+            raise FileNotFoundError(f'File not found: {previous_file}')
+        df = load(previous_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        for index, flat_row in tqdm(
+            df.iterrows(),
+            desc=f'Loading: {previous_file}',
+            total=len(df),
+        ):
+            row = prepare_row(flat_row)
+            list_keys = []
+            for key in keys:
+                value, found = search_column_value(row.nested, key)
+                if not found:
+                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
+                list_keys.append(value)
+            primary_key = tuple(list_keys)
+            #ic(key)
+            if not allow_duplicate_keys:
+                if primary_key in dict_key_to_row:
+                    ic(index)
+                    raise ValueError(f'Duplicate key: {key}')
+            dict_key_to_row[primary_key] = row
+            all_rows.append(row)
+    for modification_file in modification_files:
+        if not os.path.exists(modification_file):
+            raise FileNotFoundError(f'File not found: {modification_file}')
+        df = load(modification_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        for index, flat_row in tqdm(
+            df.iterrows(),
+            desc=f'Processing: {modification_file}',
+            total=len(df),
+        ):
+            row = prepare_row(flat_row)
+            list_keys = []
+            for key in keys:
+                value, found = search_column_value(row.nested, key)
+                if not found:
+                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
+                list_keys.append(value)
+            primary_key = tuple(list_keys)
+            #ic(key)
+            #if key not in dict_key_to_row:
+            #    dict_key_to_row[key] = row
+            #else:
+            #    dict_key_to_row[key].flat.update(row.flat)
+            if primary_key not in dict_key_to_row:
+                if ignore_not_found:
+                    list_ignored_keys.append(primary_key)
+                    continue
+                ic(index)
+                raise ValueError(f'Key not found: {key}')
+            previous_row = dict_key_to_row[primary_key]
+            #ic(previous_row)
+            #ic(previous_row.flat)
+            for key, value in row.flat.items():
+                if key.startswith('__staging__.'):
+                    continue
+                ic(key, value)
+                set_row_value(previous_row, key, value)
+                #set_row_value(previous_row, '指示追従性？', 'test')
+                #set_row_value(previous_row, 'modified', True)
+            #ic(previous_row)
+            #ic(previous_row.flat)
+            #raise
+            num_modified += 1
+    ic(num_modified)
+    if ignore_not_found:
+        ic(len(list_ignored_keys))
+        ic(list_ignored_keys)
+    if output_file:
+        #all_df = pd.DataFrame(all_rows)
+        all_df = pd.DataFrame([row.flat for row in all_rows])
+        ic('Saving to: ', output_file)
+        save(all_df, output_file)

table_processor/core/types.py

@@ -44,6 +44,7 @@ class AssignIdConfig:
     target: str
     primary: list[str]
     context: list[str] | None = None
+    reverse: bool = False
 
 @dataclasses.dataclass
 class FilterConfig:
@@ -76,6 +77,12 @@ class PickConfig:
     target: str
     source: str
 
+@dataclasses.dataclass
+class PushConfig:
+    target: str
+    source: str
+    condition: str | None = None
+
 @dataclasses.dataclass
 class SplitConfig:
     target: str

python_table_processor-0.2.29.dist-info/RECORD

@@ -1,21 +0,0 @@
-table_processor/__init__.py,sha256=i7SFej_-ia2PplNZ0Wf3m0M9G_ymn8Q72-ONSBkO8r8,54
-table_processor/cli.py,sha256=Q5eshXdgfeoiyJkFlQOCX695_6xi1dcibin0kUKOMX0,1574
-table_processor/commands/convert_tables.py,sha256=j25KL0zRl5ikrBk73jAkKn4rPmDq7SecChZxecDpILQ,2123
-table_processor/core/actions.py,sha256=-xyZp-RHjPz75JHNeKjDvdaBkDlZYAZbE2lpIJZfRDg,18524
-table_processor/core/config.py,sha256=iiHAcJlO4RyXyrXFAd2ZRU3bJBnJIFM79AdfcEMMI5o,11903
-table_processor/core/constants.py,sha256=w-1TxtpXywb6Qh8M4YWuzcCHUvCjy5zJYxaXC3RQS6A,158
-table_processor/core/convert.py,sha256=uoBHgB85VLn_Yg9ugcC0KgwTVtJHnCyEZnv2FCpbwos,12062
-table_processor/core/functions/assign_id.py,sha256=5xJuLpyWvSQ1DhzXiAOLQAwfPo9RKICzS_oka0wzHgA,2065
-table_processor/core/functions/flatten_row.py,sha256=2l-s4YXgBM58IthpfVzd7L4EGUCekSi7ss5CaM_3MUU,686
-table_processor/core/functions/get_nested_field_value.py,sha256=3u1TA2Ov5biHDLLZQLqwO0WdHMmJNsbfrI8EgstX6yA,635
-table_processor/core/functions/nest_row.py,sha256=b5AKfxE39g-NOLranQRy0dfTZq-27Or8ZlgCB8N5kFs,640
-table_processor/core/functions/search_column_value.py,sha256=GscwLaMTORHFDhdCdjzdoy46UGyFHrGwDQA2b_tZhsQ,740
-table_processor/core/functions/set_flat_field_value.py,sha256=7-BgD2yEUGyBaFaIJ18bYv82Qdy0CN8NiC5ZtsAdv44,531
-table_processor/core/functions/set_nested_field_value.py,sha256=csO0v_nW94q-M9bNL91xR-TLHPXDDxnLK8gN-0LLd6k,595
-table_processor/core/functions/set_row_value.py,sha256=wlF_nVGZY74XCzd6pBJJyti62m6y0s1kc5Q6uq9jn8o,641
-table_processor/core/types.py,sha256=jRPRut8Jr2E458SOYvB3kxKxuMWDHk5KJ8aCqmiFvMs,2365
-python_table_processor-0.2.29.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
-python_table_processor-0.2.29.dist-info/METADATA,sha256=6YjaO5gDbtQupOZiv45NHODCGYdDBX7oqKiO-WfYs4k,1099
-python_table_processor-0.2.29.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-python_table_processor-0.2.29.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
-python_table_processor-0.2.29.dist-info/RECORD,,