python-table-processor 0.2.29__tar.gz → 0.3.0__tar.gz

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-processor
-Version: 0.2.29
+Version: 0.3.0
 Summary: A table data processor
 Home-page: https://github.com/akivajp/python-table-processor
 License: MIT

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-processor"
-version = "0.2.29"
+version = "0.3.0"
 description = "A table data processor"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_processor-0.3.0/table_processor/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.3.0"
+__version_tuple__ = (0, 3, 0)

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/table_processor/cli.py

@@ -34,7 +34,21 @@ def command_convert_tables(
         )
     else:
         command_parser = parser
-    from table_converter.commands.convert_tables import setup_parser
+    from table_processor.commands.convert_tables import setup_parser
+    setup_parser(command_parser)
+    if parser is None:
+        parse_and_run(command_parser)
+
+def command_merge_tables(
+    parser: argparse.ArgumentParser|None = None,
+):
+    if parser is None:
+        command_parser = argparse.ArgumentParser(
+            description='Merge tables.'
+        )
+    else:
+        command_parser = parser
+    from table_processor.commands.merge_tables import setup_parser
     setup_parser(command_parser)
     if parser is None:
         parse_and_run(command_parser)
@@ -60,6 +74,13 @@ def main():
     setup_common_args(parser_convert_tables)
     command_convert_tables(parser_convert_tables)
 
+    parser_merge_tables = subparsers.add_parser(
+        'merge',
+        help='Merge tables.'
+    )
+    setup_common_args(parser_merge_tables)
+    command_merge_tables(parser_merge_tables)
+
     parse_and_run(parser)
 
 if __name__ == '__main__':

python_table_processor-0.3.0/table_processor/commands/merge_tables.py

@@ -0,0 +1,58 @@
+# -*- coding: utf-8 -*-
+
+import argparse
+
+from icecream import ic
+
+from .. core.merge import merge
+
+def run(
+    args: argparse.Namespace,
+):
+    merge(
+        previous_files=args.previous_files,
+        modification_files=args.modification_files,
+        keys=args.keys,
+        allow_duplicate_keys=args.allow_duplicate_keys,
+        ignore_not_found=args.ignore_not_found,
+        output_file=args.output_file,
+    )
+
+def setup_parser(
+    parser: argparse.ArgumentParser,
+):
+    parser.add_argument(
+        '--previous-files', '--previous-file', '--previous', '--old', '-P',
+        nargs='+',
+        required=True,
+        help='Previous files to merge',
+    )
+    parser.add_argument(
+        '--modification-files', '--modification', '--modify', '--modified', '--new', '-M',
+        nargs='+',
+        required=True,
+        help='Modification files to merge',
+    )
+    parser.add_argument(
+        '--keys', '-K',
+        nargs='+',
+        required=True,
+        help='Primary keys',
+    )
+    parser.add_argument(
+        '--allow-duplicate-keys', '--allow-duplicate',
+        action='store_true',
+        help='Allow duplicate keys',
+    )
+    parser.add_argument(
+        '--ignore-not-found',
+        action='store_true',
+        help='Ignore not found',
+    )
+    parser.add_argument(
+        '--output-file', '--output', '-o',
+        metavar='OUTPUT_FILE',
+        required=False,
+        help='Path to the output file.',
+    )
+    parser.set_defaults(handler=run)

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/table_processor/core/actions.py

@@ -131,10 +131,12 @@ def setup_actions_with_args(
                 context = options.get('context', None)
                 if context:
                     context = context.split(',')
+                reverse = options.get('reverse', False)
                 config.actions.append(AssignIdConfig(
                     target = target,
                     primary = [source],
                     context = context,
+                    reverse = reverse,
                 ))
                 continue
             if action_name == 'filter-empty':
@@ -488,7 +490,8 @@ def assign_format(
             params[key] = f'__{key}__undefined__'
         except:
             #ic(params)
-            ic(params.keys())
+            #ic(params.keys())
+            ic(row.flat)
             raise
     set_row_staging_value(row, config.target, formatted)
     return row

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/table_processor/core/convert.py

@@ -63,6 +63,15 @@ def register_loader(
         return loader
     return decorator
 
+def load(
+    input_file: str,
+):
+    ext = os.path.splitext(input_file)[1]
+    if ext not in dict_loaders:
+        raise ValueError(f'Unsupported file type: {ext}')
+    loader = dict_loaders[ext]
+    return loader(input_file)
+
 dict_savers: dict[str, callable] = {}
 def register_saver(
     ext: str,
@@ -72,6 +81,16 @@ def register_saver(
         return saver
     return decorator
 
+def save(
+    df: pd.DataFrame,
+    output_file: str,
+):
+    ext = os.path.splitext(output_file)[1]
+    if ext not in dict_savers:
+        raise ValueError(f'Unsupported file type: {ext}')
+    saver = dict_savers[ext]
+    saver(df, output_file)
+
 @register_loader('.csv')
 def load_csv(
     input_file: str,

python_table_processor-0.3.0/table_processor/core/functions/assign_id.py

@@ -0,0 +1,153 @@
+# -*- coding: utf-8 -*-
+
+import dataclasses
+import json
+import os
+
+from collections import OrderedDict
+from collections import defaultdict
+from typing import Mapping
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+
+# local
+
+from .. constants import (
+    STAGING_FIELD,
+)
+
+from .. config import (
+    AssignIdConfig,
+    Config,
+)
+
+from . search_column_value import search_column_value
+from . set_nested_field_value import set_nested_field_value
+from . set_row_value import set_row_staging_value
+
+from .. types import (
+    AssignIdConfig,
+    IdContextMap,
+    Row,
+)
+
+def get_key_value(
+    row: Row,
+    primary: list[str],
+    context: list[str],
+):
+    if not primary:
+        raise ValueError('Primary columns must be specified')
+    context_columns = []
+    context_values = []
+    if context:
+        for context_column in context:
+            value, found = search_column_value(row.nested, context_column)
+            if not found:
+                raise KeyError(f'Column not found: {context_column}, existing columns: {row.flat.keys()}')
+            context_columns.append(context_column)
+            context_values.append(value)
+    primary_columns = []
+    primary_values = []
+    for primary_column in primary:
+        value, found = search_column_value(row.nested, primary_column)
+        if not found:
+            raise KeyError(f'Column not found: {primary_column}, existing columns: {row.flat.keys()}')
+        primary_columns.append(primary_column)
+        primary_values.append(value)
+    context_key = (
+        tuple(context_columns),
+        tuple(context_values),
+        tuple(primary_columns),
+    )
+    primary_value = tuple(primary_values)
+    return context_key, primary_value
+
+def get_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    primary: list[str],
+    context: list[str],
+):
+    context_key, primary_value = get_key_value(
+        row=row,
+        primary=primary,
+        context=context,
+    )
+    id_map = id_context_map[context_key]
+    #ic(primary_value, context_key)
+    #ic(primary_value in id_map.dict_value_to_id)
+    if primary_value not in id_map.dict_value_to_id:
+        field_id = id_map.max_id + 1
+        id_map.max_id = field_id
+        id_map.dict_value_to_id[primary_value] = field_id
+        id_map.dict_id_to_value[field_id] = primary_value
+        id_exists = False
+    else:
+        field_id = id_map.dict_value_to_id[primary_value]
+        id_exists = True
+    return field_id, id_exists
+
+
+def set_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    primary: list[str],
+    context: list[str],
+    id_value: int,
+):
+    context_key, primary_value = get_key_value(
+        row=row,
+        primary=primary,
+        context=context,
+    )
+    id_map = id_context_map[context_key]
+    if id_value in id_map.dict_id_to_value:
+        #raise ValueError(f'ID already exists: {id_value}')
+        field_id = id_map.dict_id_to_value[id_value]
+        if field_id != primary_value:
+            raise ValueError(f'ID already exists: {id_value} for {field_id}')
+    id_map.dict_value_to_id[primary_value] = id_value
+    id_map.dict_id_to_value[id_value] = primary_value
+    id_map.max_id = max(id_map.max_id, id_value)
+    return
+
+
+def assign_id(
+    id_context_map: IdContextMap,
+    row: Row,
+    config: Config,
+):
+    if config.reverse:
+        #ic(row)
+        #ic(type(row))
+        #ic(config)
+        value, found = search_column_value(row.nested, config.target)
+        #ic(row, value, found)
+        if type(value) is str:
+            if value.isdigit():
+                value = int(value)
+        if type(value) is int:
+            field_id = value
+            set_id(
+                id_context_map=id_context_map,
+                row=row,
+                primary=config.primary,
+                context=config.context,
+                id_value=field_id,
+            )
+            #return field_id, True
+            return row
+    field_id, id_exists = get_id(
+        id_context_map=id_context_map,
+        row=row,
+        primary=config.primary,
+        context=config.context,
+    )
+    #ic(config, field_id, id_exists)
+    set_row_staging_value(row, config.target, field_id)
+    return row

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/table_processor/core/functions/get_nested_field_value.py

@@ -2,11 +2,14 @@
 
 from collections import OrderedDict
 
+from icecream import ic
+
 def get_nested_field_value(
     data: OrderedDict | list,
     field: str,
 ):
     if isinstance(data, list):
+        #ic(data, field)
         if field.isdigit():
             index = int(field)
             if index < len(data):

python_table_processor-0.3.0/table_processor/core/merge.py

@@ -0,0 +1,145 @@
+# -*- coding: utf-8 -*-
+
+import json
+import math
+import os
+
+from collections import OrderedDict
+
+from typing import (
+    Mapping,
+)
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+from tqdm.auto import tqdm
+
+# local
+
+from . functions.flatten_row import flatten_row
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.get_nested_field_value import get_nested_field_value
+from . functions.nest_row import nest_row as nest
+from . functions.search_column_value import search_column_value
+from . functions.set_nested_field_value import set_nested_field_value
+from . functions.set_row_value import (
+    set_row_value,
+    set_row_staging_value,
+)
+
+from . actions import (
+    prepare_row,
+)
+
+from . convert import (
+    load,
+    save,
+)
+
+def merge(
+    previous_files: list[str],
+    modification_files: list[str],
+    keys: list[str],
+    allow_duplicate_keys: bool = False,
+    ignore_not_found: bool = False,
+    output_file: str | None = None,
+):
+    ic.enable()
+    ic()
+    ic(previous_files)
+    ic(modification_files)
+    ic(keys)
+    dict_key_to_row = {}
+    all_rows = []
+    list_ignored_keys = []
+    num_modified = 0
+    for previous_file in previous_files:
+        if not os.path.exists(previous_file):
+            raise FileNotFoundError(f'File not found: {previous_file}')
+        df = load(previous_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        for index, flat_row in tqdm(
+            df.iterrows(),
+            desc=f'Loading: {previous_file}',
+            total=len(df),
+        ):
+            row = prepare_row(flat_row)
+            list_keys = []
+            for key in keys:
+                value, found = search_column_value(row.nested, key)
+                if not found:
+                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
+                list_keys.append(value)
+            primary_key = tuple(list_keys)
+            #ic(key)
+            if not allow_duplicate_keys:
+                if primary_key in dict_key_to_row:
+                    ic(index)
+                    raise ValueError(f'Duplicate key: {key}')
+            dict_key_to_row[primary_key] = row
+            all_rows.append(row)
+    for modification_file in modification_files:
+        if not os.path.exists(modification_file):
+            raise FileNotFoundError(f'File not found: {modification_file}')
+        df = load(modification_file)
+        # NOTE: NaN を None に変換しておかないと厄介
+        df = df.replace([np.nan], [None])
+        #ic(df)
+        ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        for index, flat_row in tqdm(
+            df.iterrows(),
+            desc=f'Processing: {modification_file}',
+            total=len(df),
+        ):
+            row = prepare_row(flat_row)
+            list_keys = []
+            for key in keys:
+                value, found = search_column_value(row.nested, key)
+                if not found:
+                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
+                list_keys.append(value)
+            primary_key = tuple(list_keys)
+            #ic(key)
+            #if key not in dict_key_to_row:
+            #    dict_key_to_row[key] = row
+            #else:
+            #    dict_key_to_row[key].flat.update(row.flat)
+            if primary_key not in dict_key_to_row:
+                if ignore_not_found:
+                    list_ignored_keys.append(primary_key)
+                    continue
+                ic(index)
+                raise ValueError(f'Key not found: {key}')
+            previous_row = dict_key_to_row[primary_key]
+            #ic(previous_row)
+            #ic(previous_row.flat)
+            for key, value in row.flat.items():
+                if key.startswith('__staging__.'):
+                    continue
+                ic(key, value)
+                set_row_value(previous_row, key, value)
+                #set_row_value(previous_row, '指示追従性？', 'test')
+                #set_row_value(previous_row, 'modified', True)
+            #ic(previous_row)
+            #ic(previous_row.flat)
+            #raise
+            num_modified += 1
+    ic(num_modified)
+    if ignore_not_found:
+        ic(len(list_ignored_keys))
+        ic(list_ignored_keys)
+    if output_file:
+        #all_df = pd.DataFrame(all_rows)
+        all_df = pd.DataFrame([row.flat for row in all_rows])
+        ic('Saving to: ', output_file)
+        save(all_df, output_file)

{python_table_processor-0.2.29 → python_table_processor-0.3.0}/table_processor/core/types.py

@@ -44,6 +44,7 @@ class AssignIdConfig:
     target: str
     primary: list[str]
     context: list[str] | None = None
+    reverse: bool = False
 
 @dataclasses.dataclass
 class FilterConfig:

python_table_processor-0.2.29/table_processor/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.29"
-__version_tuple__ = (0, 2, 29)

python_table_processor-0.2.29/table_processor/core/functions/assign_id.py

@@ -1,75 +0,0 @@
-# -*- coding: utf-8 -*-
-
-import dataclasses
-import json
-import os
-
-from collections import OrderedDict
-from collections import defaultdict
-from typing import Mapping
-
-# 3-rd party modules
-
-from icecream import ic
-import numpy as np
-import pandas as pd
-
-# local
-
-from .. constants import (
-    STAGING_FIELD,
-)
-
-from .. config import (
-    AssignIdConfig,
-    Config,
-)
-
-from . search_column_value import search_column_value
-from . set_nested_field_value import set_nested_field_value
-from . set_row_value import set_row_staging_value
-
-from .. types import (
-    AssignIdConfig,
-    IdContextMap,
-    Row,
-)
-
-def assign_id(
-    id_context_map: IdContextMap,
-    row: Row,
-    config: Config,
-):
-    context_columns = []
-    context_values = []
-    if config.context:
-        for context_column in config.context:
-            value, found = search_column_value(row.nested, context_column)
-            if not found:
-                raise KeyError(f'Column not found: {context_column}, existing columns: {row.flat.keys()}')
-            context_columns.append(context_column)
-            context_values.append(value)
-    primary_columns = []
-    primary_values = []
-    for primary_column in config.primary:
-        value, found = search_column_value(row.nested, primary_column)
-        if not found:
-            raise KeyError(f'Column not found: {primary_column}, existing columns: {row.flat.keys()}')
-        primary_columns.append(primary_column)
-        primary_values.append(value)
-    context_key = (
-        tuple(context_columns),
-        tuple(context_values),
-        tuple(primary_columns),
-    )
-    primary_value = tuple(primary_values)
-    id_map = id_context_map[context_key]
-    if primary_value not in id_map.dict_value_to_id:
-        field_id = id_map.max_id + 1
-        id_map.max_id = field_id
-        id_map.dict_value_to_id[primary_value] = field_id
-        id_map.dict_id_to_value[field_id] = primary_value
-    else:
-        field_id = id_map.dict_value_to_id[primary_value]
-    set_row_staging_value(row, config.target, field_id)
-    return row