python-table-converter 0.2.8__tar.gz → 0.2.10__tar.gz

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.8
+Version: 0.2.10
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.8"
+version = "0.2.10"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.10/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.10"
+__version_tuple__ = (0, 2, 10)

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/table_converter/commands/convert_tables.py

@@ -15,9 +15,11 @@ def run(
         config_path = args.config,
         assign_constants = args.assign_constants,
         assign_formats = args.assign_formats,
+        str_filters = args.filters,
         pickup_columns= args.pickup_columns,
         fields_to_split_by_newline = args.split_by_newline,
         fields_to_assign_ids = args.assign_ids,
+        str_omit_fields= args.omit_fields,
         output_debug = args.output_debug,
     )
 
@@ -66,6 +68,16 @@ def setup_parser(
         type=str,
         help='Field to assign formats',
     )
+    parser.add_argument(
+        '--filters', '--filter', '-f',
+        type=str,
+        help='Expression list to filter records',
+    )
+    parser.add_argument(
+        '--omit-fields', '--omit',
+        type=str,
+        help='Field to omit',
+    )
     parser.add_argument(
         '--output-debug',
         action='store_true',

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/table_converter/core/config.py

@@ -2,26 +2,39 @@
 
 from collections import OrderedDict
 import dataclasses
-from typing import Mapping
+from typing import (
+    Literal,
+    Mapping,
+)
 
 from icecream import ic
 import yaml
 
-type FlatFieldMap = Mapping[str, str]
-type FieldMap = Mapping[str, str|FieldMap]
+from . functions.flatten import (
+    FieldMap,
+    FlatFieldMap,
+    flatten,
+)
 
 @dataclasses.dataclass
 class AssignIdConfig:
     primary: list[str]
-    #given: list[str] | None = None
     context: list[str] | None = None
 
+@dataclasses.dataclass
+class FilterConfig:
+    field: str
+    operator: Literal['==', '!=', '>', '>=', '<', '<=']
+    value: str
+
 @dataclasses.dataclass
 class ProcessConfig:
     assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    #assign_ids: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_ids: Mapping[str, AssignIdConfig] = dataclasses.field(default_factory=OrderedDict)
+    #filter_eq: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    filter: list[FilterConfig] = dataclasses.field(default_factory=list)
+    omit_fields: list[str] = dataclasses.field(default_factory=list)
     split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
 
     def __setitem__(self, key, value):
@@ -32,21 +45,6 @@ class Config:
     map: FieldMap = dataclasses.field(default_factory=OrderedDict)
     process: ProcessConfig = dataclasses.field(default_factory=ProcessConfig)
 
-def flatten(
-    mapping: FieldMap,
-    parent_key: str = '',
-    new_mapping: FlatFieldMap | None = None,
-) -> FlatFieldMap:
-    if new_mapping is None:
-        new_mapping = OrderedDict()
-    for key, mapped in mapping.items():
-        new_key = f'{parent_key}.{key}' if parent_key else key
-        if isinstance(mapped, Mapping):
-            flatten(mapped, new_key, new_mapping)
-        else:
-            new_mapping[new_key] = mapped
-    return new_mapping
-
 def setup_config(
     config_path: str | None = None,
 ):
@@ -78,6 +76,7 @@ def setup_process_config(
         for process_key in [
             'assign_constants',
             'assign_formats',
+            'filter_eq',
             'split_by_newline',
         ]:
             dict_subprocess = dict_process.get(process_key)

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/table_converter/core/convert.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 
 import json
+import math
 import os
 
 from collections import OrderedDict
@@ -13,7 +14,10 @@ import pandas as pd
 
 # local
 
-from . config import setup_config
+from . config import (
+    FilterConfig,
+    setup_config,
+)
 from . constants import (
     FILE_FIELD,
     INPUT_FIELD,
@@ -24,7 +28,11 @@ from . functions.assign_id import (
     create_id_context_map,
     setup_assign_ids,
 )
+from . functions.flatten import (
+    flatten,
+)
 from . functions.get_field_value import get_field_value
+from . functions.search_column_value import search_column_value
 from . functions.set_field_value import set_field_value
 
 dict_loaders: dict[str, callable] = {}
@@ -58,9 +66,31 @@ def load_json(
 ):
     with open(input_file, 'r') as f:
         data = json.load(f)
-    df = pd.DataFrame(data)
+    if not isinstance(data, list):
+        raise ValueError(f'Invalid JSON array data: {input_file}')
+    #ic(data[0])
+    rows = []
+    for row in data:
+        new_row = flatten(row)
+        rows.append(new_row)
+    df = pd.DataFrame(rows)
     return df
 
+def nest(
+    row: OrderedDict,
+    remove_nan: bool = True,
+):
+    new_row = OrderedDict()
+    for key, value in row.items():
+        if isinstance(value, OrderedDict):
+            value = nest(value)
+        if isinstance(value, float):
+            if math.isnan(value):
+                if remove_nan:
+                    continue
+        set_field_value(new_row, key, value)
+    return new_row
+
 @register_saver('.json')
 def save_json(
     df: pd.DataFrame,
@@ -76,6 +106,9 @@ def save_json(
     #)
     #ic(df.iloc[0])
     data = df.to_dict(orient='records')
+    #ic(data[0])
+    data = [nest(row) for row in data]
+    #ic(data[0])
     with open(output_file, 'w') as f:
         json.dump(
             data,
@@ -106,34 +139,13 @@ def save_jsonl(
             )
             f.write('\n')
 
-def search_column_value(
-    row: OrderedDict,
-    column: str,
-):
-    if STAGING_FIELD in row:
-        value, found = get_field_value(row[STAGING_FIELD], column)
-        if found:
-            return value, True
-    value, found = get_field_value(row[STAGING_FIELD], column)
-    original, found = get_field_value(row, f'{STAGING_FIELD}.{INPUT_FIELD}')
-    if found:
-        value, found = get_field_value(original, column)
-        if found:
-            return value, True
-    value, found = get_field_value(row, column)
-    if found:
-        set_field_value(row, column, value)
-        return value, True
-    return None, False
-
 def map_constants(
     row: OrderedDict,
     dict_constants: OrderedDict,
 ):
     new_row = OrderedDict(row)
     for column in dict_constants.keys():
-        #set_field_value(new_row, column, dict_constants[column])
-        set_field_value(new_row, f'{STAGING_FIELD}.{column}', dict_constants[column])
+        new_row[f'{STAGING_FIELD}.{column}'] = dict_constants[column]
     return new_row
 
 def map_formats(
@@ -144,7 +156,11 @@ def map_formats(
     for column in dict_formats.keys():
         template = dict_formats[column]
         params = {}
-        params.update(row[STAGING_FIELD])
+        for key, value in row.items():
+            prefix = f'{STAGING_FIELD}.'
+            if key.startswith(prefix):
+                rest = key[len(prefix):]
+                params[rest] = value
         params.update(row)
         formatted = None
         while formatted is None:
@@ -158,7 +174,7 @@ def map_formats(
                 params[key] = f'__{key}__undefined__'
             except:
                 raise
-        set_field_value(new_row, f'{STAGING_FIELD}.{column}', formatted)
+        new_row[f'{STAGING_FIELD}.{column}'] = formatted
     return new_row
 
 def remap_columns(
@@ -169,11 +185,12 @@ def remap_columns(
     for column in dict_remap.keys():
         value, found = search_column_value(row, dict_remap[column])
         if found:
-            set_field_value(new_row, column, value)
+            new_row[column] = value
     for column in row.keys():
-        if column == STAGING_FIELD:
-            # NOTE: Ignore debug fields
-            set_field_value(new_row, column, row[column])
+        prefix = f'{STAGING_FIELD}.'
+        if column.startswith(prefix):
+            # NOTE: Leave staging fields as is
+            new_row[column] = row[column]
     return new_row
 
 def apply_fields_split_by_newline(
@@ -183,21 +200,40 @@ def apply_fields_split_by_newline(
     new_row = OrderedDict(row)
     for column in dict_fields:
         value, found = search_column_value(row, dict_fields[column])
-        #ic(value, found)
         if found:
             if isinstance(value, str):
                 new_value = value.split('\n')
-                set_field_value(new_row, f'{STAGING_FIELD}.{column}', new_value)
+                new_row[f'{STAGING_FIELD}.{column}'] = new_value
             else:
-                set_field_value(new_row, f'{STAGING_FIELD}.{column}', value)
+                new_row[f'{STAGING_FIELD}.{column}'] = value
     return new_row
 
+def filter_row(
+    row: OrderedDict,
+    list_filters: list[FilterConfig],
+):
+    for config in list_filters:
+        value, found = search_column_value(row, config.field)
+        if config.operator == '==':
+            if not found:
+                return False
+            if str(value) != str(config.value):
+                return False
+        elif config.operator == '!=':
+            if str(value) == str(config.value):
+                return False
+        else:
+            raise ValueError(f'Unsupported operator: {config.operator}')
+    return True
+
 def convert(
     input_files: list[str],
     output_file: str | None = None,
     config_path: str | None = None,
     assign_constants: str | None = None,
     assign_formats: str | None = None,
+    str_filters: str | None = None,
+    str_omit_fields: str | None = None,
     pickup_columns: str | None = None,
     fields_to_split_by_newline: str | None = None,
     fields_to_assign_ids: str | None = None,
@@ -242,6 +278,29 @@ def convert(
                 config.process.split_by_newline[dst] = src
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
+    if str_filters:
+        fields = str_filters.split(',')
+        for field in fields:
+            if '==' in field:
+                column, value = field.split('==')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '==',
+                    value = value,
+                ))
+            elif '!=' in field:
+                column, value = field.split('!=')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '!=',
+                    value = value,
+                ))
+            else:
+                raise ValueError(f'Invalid filter eq: {field}')
+    if str_omit_fields:
+        fields = str_omit_fields.split(',')
+        for field in fields:
+            config.process.omit_fields.append(field)
     if fields_to_assign_ids:
         setup_assign_ids(config, fields_to_assign_ids)
     if output_file:
@@ -263,38 +322,49 @@ def convert(
         # NOTE: NaN を None に変換しておかないと厄介
         df = df.replace([np.nan], [None])
         #ic(df)
-        ic(len(df))
-        ic(df.columns)
-        ic(df.iloc[0])
-        new_rows = []
-        for index, row in df.iterrows():
-            orig = OrderedDict(row)
-            new_row = OrderedDict(row)
-            if STAGING_FIELD not in new_row:
-                set_field_value(new_row, f'{STAGING_FIELD}.{INPUT_FIELD}', orig)
-                set_field_value(new_row, f'{STAGING_FIELD}.{FILE_FIELD}', input_file)
+        #ic(len(df))
+        #ic(df.columns)
+        #ic(df.iloc[0])
+        #new_rows = []
+        new_flat_rows = []
+        for index, flat_row in df.iterrows():
+            orig = OrderedDict(flat_row)
+            new_flat_row = OrderedDict(flat_row)
+            new_nested_row = nest(new_flat_row)
+            if STAGING_FIELD not in new_nested_row:
+                new_flat_row[f'{STAGING_FIELD}.{FILE_FIELD}'] = input_file
+                for key, value in orig.items():
+                    new_flat_row[f'{STAGING_FIELD}.{INPUT_FIELD}.{key}'] = value
             if config.process.assign_constants:
-                new_row = map_constants(new_row, config.process.assign_constants)
+                new_flat_row = map_constants(new_flat_row, config.process.assign_constants)
             if config.map:
-                new_row = remap_columns(new_row, config.map)
+                new_flat_row = remap_columns(new_flat_row, config.map)
             if config.process.split_by_newline:
-                new_row = apply_fields_split_by_newline(new_row, config.process.split_by_newline)
+                new_flat_row = apply_fields_split_by_newline(new_flat_row, config.process.split_by_newline)
             if config.process.assign_ids:
-                new_row = assign_id(new_row, config.process.assign_ids, id_context_map)
+                new_flat_row = assign_id(new_flat_row, config.process.assign_ids, id_context_map)
             if config.process.assign_formats:
-                new_row = map_formats(new_row, config.process.assign_formats)
+                new_flat_row = map_formats(new_flat_row, config.process.assign_formats)
             if config.map:
-                new_row = remap_columns(new_row, config.map)
+                new_flat_row = remap_columns(new_flat_row, config.map)
+            if config.process.filter:
+                if not filter_row(new_flat_row, config.process.filter):
+                    continue
+            if config.process.omit_fields:
+                for field in config.process.omit_fields:
+                    new_flat_row.pop(field, None)
             if not output_debug:
-                new_row.pop(STAGING_FIELD, None)
-            new_rows.append(new_row)
-        new_df = pd.DataFrame(new_rows)
+                for key in list(new_flat_row.keys()):
+                    if key.startswith(STAGING_FIELD):
+                        new_flat_row.pop(key)
+            new_flat_rows.append(new_flat_row)
+        new_df = pd.DataFrame(new_flat_rows)
         df_list.append(new_df)
     all_df = pd.concat(df_list)
     #ic(all_df)
     ic(len(all_df))
-    ic(all_df.columns)
-    ic(all_df.iloc[0])
+    #ic(all_df.columns)
+    #ic(all_df.iloc[0])
     if output_file:
         ic('Saing to: ', output_file)
         saver(all_df, output_file)

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/table_converter/core/functions/assign_id.py

@@ -16,7 +16,11 @@ import pandas as pd
 
 # local
 
-from ..config import (
+from .. constants import (
+    STAGING_FIELD,
+)
+
+from .. config import (
     AssignIdConfig,
     Config,
 )
@@ -87,7 +91,7 @@ def assign_id(
             id_map.dict_id_to_value[field_id] = primary_value
         else:
             field_id = id_map.dict_value_to_id[primary_value]
-        set_field_value(new_row, f'__debug__.{column}', field_id)
+        new_row[f'{STAGING_FIELD}.{column}'] = field_id
     return new_row
 
 def setup_assign_ids(

python_table_converter-0.2.10/table_converter/core/functions/flatten.py

@@ -0,0 +1,24 @@
+'''
+This module contains the function to flatten a nested dictionary.
+'''
+
+from collections import OrderedDict
+from typing import Mapping
+
+type FlatFieldMap = Mapping[str]
+type FieldMap = Mapping[str, str|FieldMap]
+
+def flatten(
+    mapping: FieldMap,
+    parent_key: str = '',
+    new_mapping: FlatFieldMap | None = None,
+) -> FlatFieldMap:
+    if new_mapping is None:
+        new_mapping = OrderedDict()
+    for key, mapped in mapping.items():
+        new_key = f'{parent_key}.{key}' if parent_key else key
+        if isinstance(mapped, Mapping):
+            flatten(mapped, new_key, new_mapping)
+        else:
+            new_mapping[new_key] = mapped
+    return new_mapping

python_table_converter-0.2.10/table_converter/core/functions/search_column_value.py

@@ -0,0 +1,25 @@
+'''
+This function is used to search for a column value in a row. It will first search in the '__debug__' field, then in the '__debug__.__original__' field, and finally in the row itself. If the value is found, it will be set in the row and returned.
+'''
+
+from .. constants import (
+    INPUT_FIELD,
+    STAGING_FIELD,
+)
+
+from collections import OrderedDict
+
+def search_column_value(
+    row: OrderedDict,
+    column: str,
+):
+    if f'{STAGING_FIELD}.{column}' in row:
+        value = row[f'{STAGING_FIELD}.{column}']
+        return value, True
+    if f'{STAGING_FIELD}.{INPUT_FIELD}.{column}' in row:
+        value = row[f'{STAGING_FIELD}.{INPUT_FIELD}.{column}']
+        return value, True
+    if column in row:
+        value = row[column]
+        return value, True
+    return None, False

{python_table_converter-0.2.8 → python_table_converter-0.2.10}/table_converter/core/functions/set_field_value.py

@@ -3,6 +3,7 @@ Set the value of a field in a nested dictionary.
 '''
 
 from collections import OrderedDict
+from icecream import ic
 
 def set_field_value(
     data: OrderedDict,

python_table_converter-0.2.8/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.8"
-__version_tuple__ = (0, 2, 8)

python_table_converter-0.2.8/table_converter/core/functions/search_column_value.py

@@ -1,30 +0,0 @@
-'''
-This function is used to search for a column value in a row. It will first search in the '__debug__' field, then in the '__debug__.__original__' field, and finally in the row itself. If the value is found, it will be set in the row and returned.
-'''
-
-from .. constants import STAGING_FIELD
-
-from collections import OrderedDict
-
-from . get_field_value import get_field_value
-from . set_field_value import set_field_value
-
-def search_column_value(
-    row: OrderedDict,
-    column: str,
-):
-    if STAGING_FIELD in row:
-        value, found = get_field_value(row[STAGING_FIELD], column)
-        if found:
-            return value, True
-    value, found = get_field_value(row[STAGING_FIELD], column)
-    original, found = get_field_value(row, f'{STAGING_FIELD}.__original__')
-    if found:
-        value, found = get_field_value(original, column)
-        if found:
-            return value, True
-    value, found = get_field_value(row, column)
-    if found:
-        set_field_value(row, column, value)
-        return value, True
-    return None, False