python-table-converter 0.2.9__tar.gz → 0.2.11__tar.gz

{python_table_converter-0.2.9 → python_table_converter-0.2.11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.9
+Version: 0.2.11
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.9 → python_table_converter-0.2.11}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.9"
+version = "0.2.11"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.11/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.11"
+__version_tuple__ = (0, 2, 11)

{python_table_converter-0.2.9 → python_table_converter-0.2.11}/table_converter/commands/convert_tables.py

@@ -15,9 +15,11 @@ def run(
         config_path = args.config,
         assign_constants = args.assign_constants,
         assign_formats = args.assign_formats,
+        str_filters = args.filters,
         pickup_columns= args.pickup_columns,
         fields_to_split_by_newline = args.split_by_newline,
         fields_to_assign_ids = args.assign_ids,
+        str_omit_fields= args.omit_fields,
         output_debug = args.output_debug,
     )
 
@@ -66,6 +68,16 @@ def setup_parser(
         type=str,
         help='Field to assign formats',
     )
+    parser.add_argument(
+        '--filters', '--filter', '-f',
+        type=str,
+        help='Expression list to filter records',
+    )
+    parser.add_argument(
+        '--omit-fields', '--omit',
+        type=str,
+        help='Field to omit',
+    )
     parser.add_argument(
         '--output-debug',
         action='store_true',

{python_table_converter-0.2.9 → python_table_converter-0.2.11}/table_converter/core/config.py

@@ -2,7 +2,10 @@
 
 from collections import OrderedDict
 import dataclasses
-from typing import Mapping
+from typing import (
+    Literal,
+    Mapping,
+)
 
 from icecream import ic
 import yaml
@@ -18,13 +21,27 @@ class AssignIdConfig:
     primary: list[str]
     context: list[str] | None = None
 
+@dataclasses.dataclass
+class FilterConfig:
+    field: str
+    operator: Literal['==', '!=', '>', '>=', '<', '<=']
+    value: str
+
+@dataclasses.dataclass
+class SplitConfig:
+    field: str
+    delimiter: str
+
 @dataclasses.dataclass
 class ProcessConfig:
     assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    #assign_ids: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_ids: Mapping[str, AssignIdConfig] = dataclasses.field(default_factory=OrderedDict)
-    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    #filter_eq: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    filter: list[FilterConfig] = dataclasses.field(default_factory=list)
+    omit_fields: list[str] = dataclasses.field(default_factory=list)
+    #split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    split: Mapping[str, SplitConfig] = dataclasses.field(default_factory=OrderedDict)
 
     def __setitem__(self, key, value):
         setattr(self, key, value)
@@ -65,12 +82,12 @@ def setup_process_config(
         for process_key in [
             'assign_constants',
             'assign_formats',
-            'split_by_newline',
         ]:
             dict_subprocess = dict_process.get(process_key)
             if isinstance(dict_subprocess, Mapping):
                 config.process[process_key] = flatten(loaded['process'][process_key])
         setup_process_assign_ids_config(config, dict_process)
+        setup_process_split_config(config, dict_process)
 
 def setup_process_assign_ids_config(
     config: Config,
@@ -108,3 +125,39 @@ def setup_process_assign_ids_config(
                 raise ValueError(
                     f'Unsupported assign_ids value type: {type(value)}'
                 )
+
+def setup_process_split_config(
+    config: Config,
+    dict_process: Mapping,
+):
+    dict_subprocess = dict_process.get('split')
+    if isinstance(dict_subprocess, Mapping):
+        for key, value in dict_subprocess.items():
+            if isinstance(value, Mapping):
+                field = value.get('field')
+                if not field:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('field'))
+                    raise ValueError(
+                        'Field is required for split.'
+                    )
+                delimiter = value.get('delimiter')
+                if not delimiter:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('delimiter'))
+                    raise ValueError(
+                        'Delimiter is required for split.'
+                    )
+                config.process.split[key] = SplitConfig(
+                    field = field,
+                    delimiter = delimiter,
+                )
+            else:
+                ic.enable()
+                ic(value)
+                ic(type(value))
+                raise ValueError(
+                    f'Unsupported assign_ids value type: {type(value)}'
+                )

{python_table_converter-0.2.9 → python_table_converter-0.2.11}/table_converter/core/convert.py

@@ -14,7 +14,11 @@ import pandas as pd
 
 # local
 
-from . config import setup_config
+from . config import (
+    FilterConfig,
+    SplitConfig,
+    setup_config,
+)
 from . constants import (
     FILE_FIELD,
     INPUT_FIELD,
@@ -65,7 +69,7 @@ def load_json(
         data = json.load(f)
     if not isinstance(data, list):
         raise ValueError(f'Invalid JSON array data: {input_file}')
-    ic(data[0])
+    #ic(data[0])
     rows = []
     for row in data:
         new_row = flatten(row)
@@ -103,9 +107,9 @@ def save_json(
     #)
     #ic(df.iloc[0])
     data = df.to_dict(orient='records')
-    ic(data[0])
+    #ic(data[0])
     data = [nest(row) for row in data]
-    ic(data[0])
+    #ic(data[0])
     with open(output_file, 'w') as f:
         json.dump(
             data,
@@ -190,27 +194,50 @@ def remap_columns(
             new_row[column] = row[column]
     return new_row
 
-def apply_fields_split_by_newline(
+def split_fields(
     row: OrderedDict,
-    dict_fields: OrderedDict,
+    dict_config: list[SplitConfig],
 ):
     new_row = OrderedDict(row)
-    for column in dict_fields:
-        value, found = search_column_value(row, dict_fields[column])
+    for dst, config in dict_config.items():
+        value, found = search_column_value(row, config.field)
         if found:
             if isinstance(value, str):
-                new_value = value.split('\n')
-                new_row[f'{STAGING_FIELD}.{column}'] = new_value
+                new_value = value.split(config.delimiter)
+                new_value = list(filter(None, new_value))
+                if not new_value:
+                    ic(dst, config, value)
+                new_row[f'{STAGING_FIELD}.{dst}'] = new_value
             else:
-                new_row[f'{STAGING_FIELD}.{column}'] = value
+                new_row[f'{STAGING_FIELD}.{dst}'] = value
     return new_row
 
+def filter_row(
+    row: OrderedDict,
+    list_filters: list[FilterConfig],
+):
+    for config in list_filters:
+        value, found = search_column_value(row, config.field)
+        if config.operator == '==':
+            if not found:
+                return False
+            if str(value) != str(config.value):
+                return False
+        elif config.operator == '!=':
+            if str(value) == str(config.value):
+                return False
+        else:
+            raise ValueError(f'Unsupported operator: {config.operator}')
+    return True
+
 def convert(
     input_files: list[str],
     output_file: str | None = None,
     config_path: str | None = None,
     assign_constants: str | None = None,
     assign_formats: str | None = None,
+    str_filters: str | None = None,
+    str_omit_fields: str | None = None,
     pickup_columns: str | None = None,
     fields_to_split_by_newline: str | None = None,
     fields_to_assign_ids: str | None = None,
@@ -252,9 +279,36 @@ def convert(
         for field in fields:
             if '=' in field:
                 dst, src = field.split('=')
-                config.process.split_by_newline[dst] = src
+                #config.process.split_by_newline[dst] = src
+                config.process.split[dst] = SplitConfig(
+                    field = src,
+                    delimiter = '\n',
+                )
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
+    if str_filters:
+        fields = str_filters.split(',')
+        for field in fields:
+            if '==' in field:
+                column, value = field.split('==')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '==',
+                    value = value,
+                ))
+            elif '!=' in field:
+                column, value = field.split('!=')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '!=',
+                    value = value,
+                ))
+            else:
+                raise ValueError(f'Invalid filter eq: {field}')
+    if str_omit_fields:
+        fields = str_omit_fields.split(',')
+        for field in fields:
+            config.process.omit_fields.append(field)
     if fields_to_assign_ids:
         setup_assign_ids(config, fields_to_assign_ids)
     if output_file:
@@ -293,24 +347,32 @@ def convert(
                 new_flat_row = map_constants(new_flat_row, config.process.assign_constants)
             if config.map:
                 new_flat_row = remap_columns(new_flat_row, config.map)
-            if config.process.split_by_newline:
-                new_flat_row = apply_fields_split_by_newline(new_flat_row, config.process.split_by_newline)
+            if config.process.split:
+                new_flat_row = split_fields(new_flat_row, config.process.split)
             if config.process.assign_ids:
                 new_flat_row = assign_id(new_flat_row, config.process.assign_ids, id_context_map)
             if config.process.assign_formats:
                 new_flat_row = map_formats(new_flat_row, config.process.assign_formats)
             if config.map:
                 new_flat_row = remap_columns(new_flat_row, config.map)
+            if config.process.filter:
+                if not filter_row(new_flat_row, config.process.filter):
+                    continue
+            if config.process.omit_fields:
+                for field in config.process.omit_fields:
+                    new_flat_row.pop(field, None)
             if not output_debug:
-                new_flat_row.pop(STAGING_FIELD, None)
+                for key in list(new_flat_row.keys()):
+                    if key.startswith(STAGING_FIELD):
+                        new_flat_row.pop(key)
             new_flat_rows.append(new_flat_row)
         new_df = pd.DataFrame(new_flat_rows)
         df_list.append(new_df)
     all_df = pd.concat(df_list)
     #ic(all_df)
     ic(len(all_df))
-    ic(all_df.columns)
-    ic(all_df.iloc[0])
+    #ic(all_df.columns)
+    #ic(all_df.iloc[0])
     if output_file:
         ic('Saing to: ', output_file)
         saver(all_df, output_file)

python_table_converter-0.2.9/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.9"
-__version_tuple__ = (0, 2, 9)