python-table-converter 0.2.14__tar.gz → 0.2.17__tar.gz

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.14
+Version: 0.2.17
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.14"
+version = "0.2.17"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.17/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.17"
+__version_tuple__ = (0, 2, 17)

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/table_converter/commands/convert_tables.py

@@ -12,6 +12,7 @@ def run(
     convert(
         input_files = args.input_files,
         output_file = args.output_file,
+        output_file_filtered_out = args.output_file_filtered_out,
         config_path = args.config,
         list_actions = args.do_actions,
         list_pick_columns = args.pick_columns,
@@ -34,6 +35,12 @@ def setup_parser(
         required=True,
         help='Path to the output file.'
     )
+    parser.add_argument(
+        '--output-file-filtered-out', '--output-filtered-out', '-f',
+        metavar='OUTPUT_FILE_FILTERED_OUT',
+        required=False,
+        help='Path to the output file for filtered out rows.',
+    )
     parser.add_argument(
         '--config', '-c',
         type=str,

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/table_converter/core/actions.py

@@ -2,6 +2,8 @@
 Actions are used to transform the data in the table.
 '''
 
+import re
+
 from collections import OrderedDict
 from dataclasses import dataclass
 from typing import (
@@ -190,6 +192,14 @@ def setup_filter_action(
             value = value.strip(),
         ))
         return config
+    if '=~' in str_filter:
+        field, value = str_filter.split('=~')
+        config.actions.append(FilterConfig(
+            field = field.strip(),
+            operator = '=~',
+            value = value.strip(),
+        ))
+        return config
     raise ValueError(
         f'Unsupported filter: {str_filter}'
     )
@@ -383,6 +393,11 @@ def filter_row(
     elif config.operator == '!=':
         if str(value) == str(config.value) or value == config.value:
             return False
+    elif config.operator == '=~':
+        if not found:
+            return False
+        if not re.search(config.value, value):
+            return False
     elif config.operator == 'not-in':
         if isinstance(config.value, list):
             if value in config.value:

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/table_converter/core/convert.py

@@ -119,6 +119,18 @@ def save_json(
             ensure_ascii=False,
         )
 
+@register_loader('.jsonl')
+def load_jsonl(
+    input_file: str,
+):
+    rows = []
+    with open(input_file, 'r') as f:
+        for line in f:
+            row = json.loads(line)
+            rows.append(row)
+    df = pd.DataFrame(rows)
+    return df
+
 @register_saver('.jsonl')
 def save_jsonl(
     df: pd.DataFrame,
@@ -244,6 +256,7 @@ def assign_length(
 def convert(
     input_files: list[str],
     output_file: str | None = None,
+    output_file_filtered_out: str | None = None,
     config_path: str | None = None,
     output_debug: bool = False,
     list_actions: list[str] | None = None,
@@ -254,6 +267,7 @@ def convert(
     ic()
     ic(input_files)
     df_list = []
+    row_list_filtered_out = []
     global_status = GlobalStatus()
     config = setup_config(config_path)
     ic(config)
@@ -300,9 +314,12 @@ def convert(
             if config.actions:
                 new_row = do_actions(global_status, row, config.actions)
                 if new_row is None:
-                    if verbose:
+                    if not output_debug:
                         pop_row_staging(row)
+                    if verbose:
                         ic('Filtered out: ', row.flat)
+                    if output_file_filtered_out:
+                        row_list_filtered_out.append(row.flat)
                     continue
                 row = new_row
             if config.pick:
@@ -320,3 +337,7 @@ def convert(
     if output_file:
         ic('Saing to: ', output_file)
         saver(all_df, output_file)
+    if row_list_filtered_out:
+        df_filtered_out = pd.DataFrame(row_list_filtered_out)
+        ic('Saving filtered out to: ', output_file_filtered_out)
+        saver(df_filtered_out, output_file_filtered_out)

{python_table_converter-0.2.14 → python_table_converter-0.2.17}/table_converter/core/types.py

@@ -40,7 +40,7 @@ class AssignIdConfig:
 @dataclasses.dataclass
 class FilterConfig:
     field: str
-    operator: Literal['==', '!=', '>', '>=', '<', '<=', 'not-in']
+    operator: Literal['==', '!=', '>', '>=', '<', '<=', '=~', 'not-in']
     value: str | list[str]
 
 @dataclasses.dataclass

python_table_converter-0.2.14/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.14"
-__version_tuple__ = (0, 2, 14)