python-table-converter 0.2.9__tar.gz → 0.2.10__tar.gz

{python_table_converter-0.2.9 → python_table_converter-0.2.10}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.9
+Version: 0.2.10
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.9 → python_table_converter-0.2.10}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.9"
+version = "0.2.10"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.10/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.10"
+__version_tuple__ = (0, 2, 10)

{python_table_converter-0.2.9 → python_table_converter-0.2.10}/table_converter/commands/convert_tables.py

@@ -15,9 +15,11 @@ def run(
         config_path = args.config,
         assign_constants = args.assign_constants,
         assign_formats = args.assign_formats,
+        str_filters = args.filters,
         pickup_columns= args.pickup_columns,
         fields_to_split_by_newline = args.split_by_newline,
         fields_to_assign_ids = args.assign_ids,
+        str_omit_fields= args.omit_fields,
         output_debug = args.output_debug,
     )
 
@@ -66,6 +68,16 @@ def setup_parser(
         type=str,
         help='Field to assign formats',
     )
+    parser.add_argument(
+        '--filters', '--filter', '-f',
+        type=str,
+        help='Expression list to filter records',
+    )
+    parser.add_argument(
+        '--omit-fields', '--omit',
+        type=str,
+        help='Field to omit',
+    )
     parser.add_argument(
         '--output-debug',
         action='store_true',

{python_table_converter-0.2.9 → python_table_converter-0.2.10}/table_converter/core/config.py

@@ -2,7 +2,10 @@
 
 from collections import OrderedDict
 import dataclasses
-from typing import Mapping
+from typing import (
+    Literal,
+    Mapping,
+)
 
 from icecream import ic
 import yaml
@@ -18,12 +21,20 @@ class AssignIdConfig:
     primary: list[str]
     context: list[str] | None = None
 
+@dataclasses.dataclass
+class FilterConfig:
+    field: str
+    operator: Literal['==', '!=', '>', '>=', '<', '<=']
+    value: str
+
 @dataclasses.dataclass
 class ProcessConfig:
     assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    #assign_ids: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     assign_ids: Mapping[str, AssignIdConfig] = dataclasses.field(default_factory=OrderedDict)
+    #filter_eq: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    filter: list[FilterConfig] = dataclasses.field(default_factory=list)
+    omit_fields: list[str] = dataclasses.field(default_factory=list)
     split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
 
     def __setitem__(self, key, value):
@@ -65,6 +76,7 @@ def setup_process_config(
         for process_key in [
             'assign_constants',
             'assign_formats',
+            'filter_eq',
             'split_by_newline',
         ]:
             dict_subprocess = dict_process.get(process_key)

{python_table_converter-0.2.9 → python_table_converter-0.2.10}/table_converter/core/convert.py

@@ -14,7 +14,10 @@ import pandas as pd
 
 # local
 
-from . config import setup_config
+from . config import (
+    FilterConfig,
+    setup_config,
+)
 from . constants import (
     FILE_FIELD,
     INPUT_FIELD,
@@ -65,7 +68,7 @@ def load_json(
         data = json.load(f)
     if not isinstance(data, list):
         raise ValueError(f'Invalid JSON array data: {input_file}')
-    ic(data[0])
+    #ic(data[0])
     rows = []
     for row in data:
         new_row = flatten(row)
@@ -103,9 +106,9 @@ def save_json(
     #)
     #ic(df.iloc[0])
     data = df.to_dict(orient='records')
-    ic(data[0])
+    #ic(data[0])
     data = [nest(row) for row in data]
-    ic(data[0])
+    #ic(data[0])
     with open(output_file, 'w') as f:
         json.dump(
             data,
@@ -205,12 +208,32 @@ def apply_fields_split_by_newline(
                 new_row[f'{STAGING_FIELD}.{column}'] = value
     return new_row
 
+def filter_row(
+    row: OrderedDict,
+    list_filters: list[FilterConfig],
+):
+    for config in list_filters:
+        value, found = search_column_value(row, config.field)
+        if config.operator == '==':
+            if not found:
+                return False
+            if str(value) != str(config.value):
+                return False
+        elif config.operator == '!=':
+            if str(value) == str(config.value):
+                return False
+        else:
+            raise ValueError(f'Unsupported operator: {config.operator}')
+    return True
+
 def convert(
     input_files: list[str],
     output_file: str | None = None,
     config_path: str | None = None,
     assign_constants: str | None = None,
     assign_formats: str | None = None,
+    str_filters: str | None = None,
+    str_omit_fields: str | None = None,
     pickup_columns: str | None = None,
     fields_to_split_by_newline: str | None = None,
     fields_to_assign_ids: str | None = None,
@@ -255,6 +278,29 @@ def convert(
                 config.process.split_by_newline[dst] = src
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
+    if str_filters:
+        fields = str_filters.split(',')
+        for field in fields:
+            if '==' in field:
+                column, value = field.split('==')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '==',
+                    value = value,
+                ))
+            elif '!=' in field:
+                column, value = field.split('!=')
+                config.process.filter.append(FilterConfig(
+                    field = column,
+                    operator = '!=',
+                    value = value,
+                ))
+            else:
+                raise ValueError(f'Invalid filter eq: {field}')
+    if str_omit_fields:
+        fields = str_omit_fields.split(',')
+        for field in fields:
+            config.process.omit_fields.append(field)
     if fields_to_assign_ids:
         setup_assign_ids(config, fields_to_assign_ids)
     if output_file:
@@ -301,16 +347,24 @@ def convert(
                 new_flat_row = map_formats(new_flat_row, config.process.assign_formats)
             if config.map:
                 new_flat_row = remap_columns(new_flat_row, config.map)
+            if config.process.filter:
+                if not filter_row(new_flat_row, config.process.filter):
+                    continue
+            if config.process.omit_fields:
+                for field in config.process.omit_fields:
+                    new_flat_row.pop(field, None)
             if not output_debug:
-                new_flat_row.pop(STAGING_FIELD, None)
+                for key in list(new_flat_row.keys()):
+                    if key.startswith(STAGING_FIELD):
+                        new_flat_row.pop(key)
             new_flat_rows.append(new_flat_row)
         new_df = pd.DataFrame(new_flat_rows)
         df_list.append(new_df)
     all_df = pd.concat(df_list)
     #ic(all_df)
     ic(len(all_df))
-    ic(all_df.columns)
-    ic(all_df.iloc[0])
+    #ic(all_df.columns)
+    #ic(all_df.iloc[0])
     if output_file:
         ic('Saing to: ', output_file)
         saver(all_df, output_file)

python_table_converter-0.2.9/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.9"
-__version_tuple__ = (0, 2, 9)