python-table-converter 0.2.10__tar.gz → 0.2.12__tar.gz

{python_table_converter-0.2.10 → python_table_converter-0.2.12}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.10
+Version: 0.2.12
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.10 → python_table_converter-0.2.12}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.10"
+version = "0.2.12"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.12/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.12"
+__version_tuple__ = (0, 2, 12)

{python_table_converter-0.2.10 → python_table_converter-0.2.12}/table_converter/core/config.py

@@ -24,8 +24,14 @@ class AssignIdConfig:
 @dataclasses.dataclass
 class FilterConfig:
     field: str
-    operator: Literal['==', '!=', '>', '>=', '<', '<=']
-    value: str
+    operator: Literal['==', '!=', '>', '>=', '<', '<=', 'not-in']
+    #value: str
+    value: str | list[str]
+
+@dataclasses.dataclass
+class SplitConfig:
+    field: str
+    delimiter: str
 
 @dataclasses.dataclass
 class ProcessConfig:
@@ -35,7 +41,8 @@ class ProcessConfig:
     #filter_eq: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
     filter: list[FilterConfig] = dataclasses.field(default_factory=list)
     omit_fields: list[str] = dataclasses.field(default_factory=list)
-    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    #split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    split: Mapping[str, SplitConfig] = dataclasses.field(default_factory=OrderedDict)
 
     def __setitem__(self, key, value):
         setattr(self, key, value)
@@ -76,13 +83,13 @@ def setup_process_config(
         for process_key in [
             'assign_constants',
             'assign_formats',
-            'filter_eq',
-            'split_by_newline',
         ]:
             dict_subprocess = dict_process.get(process_key)
             if isinstance(dict_subprocess, Mapping):
                 config.process[process_key] = flatten(loaded['process'][process_key])
         setup_process_assign_ids_config(config, dict_process)
+        setup_process_filter_config(config, dict_process)
+        setup_process_split_config(config, dict_process)
 
 def setup_process_assign_ids_config(
     config: Config,
@@ -120,3 +127,87 @@ def setup_process_assign_ids_config(
                 raise ValueError(
                     f'Unsupported assign_ids value type: {type(value)}'
                 )
+
+def setup_process_filter_config(
+    config: Config,
+    dict_process: Mapping,
+):
+    list_subprocess = dict_process.get('filter')
+    if not isinstance(list_subprocess, list):
+        raise ValueError(
+            'Filter must be a list.'
+        )
+    for item in list_subprocess:
+        if isinstance(item, Mapping):
+            field = item.get('field')
+            if not field:
+                ic.enable()
+                ic(item)
+                ic(item.get('field'))
+                raise ValueError(
+                    'Field is required for filter.'
+                )
+            operator = item.get('operator')
+            if not operator:
+                ic.enable()
+                ic(item)
+                ic(item.get('operator'))
+                raise ValueError(
+                    'Operator is required for filter.'
+                )
+            value = item.get('value')
+            if not value:
+                ic.enable()
+                ic(item)
+                ic(item.get('value'))
+                raise ValueError(
+                    'Value is required for filter.'
+                )
+            config.process.filter.append(FilterConfig(
+                field = field,
+                operator = operator,
+                value = value,
+            ))
+        else:
+            ic.enable()
+            ic(item)
+            ic(type(item))
+            raise ValueError(
+                f'Unsupported filter item type: {type(item)}'
+            )
+
+def setup_process_split_config(
+    config: Config,
+    dict_process: Mapping,
+):
+    dict_subprocess = dict_process.get('split')
+    if isinstance(dict_subprocess, Mapping):
+        for key, value in dict_subprocess.items():
+            if isinstance(value, Mapping):
+                field = value.get('field')
+                if not field:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('field'))
+                    raise ValueError(
+                        'Field is required for split.'
+                    )
+                delimiter = value.get('delimiter')
+                if not delimiter:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('delimiter'))
+                    raise ValueError(
+                        'Delimiter is required for split.'
+                    )
+                config.process.split[key] = SplitConfig(
+                    field = field,
+                    delimiter = delimiter,
+                )
+            else:
+                ic.enable()
+                ic(value)
+                ic(type(value))
+                raise ValueError(
+                    f'Unsupported assign_ids value type: {type(value)}'
+                )

{python_table_converter-0.2.10 → python_table_converter-0.2.12}/table_converter/core/convert.py

@@ -16,6 +16,7 @@ import pandas as pd
 
 from . config import (
     FilterConfig,
+    SplitConfig,
     setup_config,
 )
 from . constants import (
@@ -193,19 +194,22 @@ def remap_columns(
             new_row[column] = row[column]
     return new_row
 
-def apply_fields_split_by_newline(
+def split_fields(
     row: OrderedDict,
-    dict_fields: OrderedDict,
+    dict_config: list[SplitConfig],
 ):
     new_row = OrderedDict(row)
-    for column in dict_fields:
-        value, found = search_column_value(row, dict_fields[column])
+    for dst, config in dict_config.items():
+        value, found = search_column_value(row, config.field)
         if found:
             if isinstance(value, str):
-                new_value = value.split('\n')
-                new_row[f'{STAGING_FIELD}.{column}'] = new_value
+                new_value = value.split(config.delimiter)
+                new_value = list(filter(None, new_value))
+                if not new_value:
+                    ic(dst, config, value)
+                new_row[f'{STAGING_FIELD}.{dst}'] = new_value
             else:
-                new_row[f'{STAGING_FIELD}.{column}'] = value
+                new_row[f'{STAGING_FIELD}.{dst}'] = value
     return new_row
 
 def filter_row(
@@ -217,11 +221,19 @@ def filter_row(
         if config.operator == '==':
             if not found:
                 return False
-            if str(value) != str(config.value):
+            if value != config.value and str(value) != str(config.value):
                 return False
         elif config.operator == '!=':
-            if str(value) == str(config.value):
+            if str(value) == str(config.value) or value == config.value:
                 return False
+        elif config.operator == 'not-in':
+            if isinstance(config.value, list):
+                if value in config.value:
+                    return False
+                if str(value) in config.value:
+                    return False
+            else:
+                raise ValueError(f'Unsupported filter value type: type{config.value}')
         else:
             raise ValueError(f'Unsupported operator: {config.operator}')
     return True
@@ -275,7 +287,11 @@ def convert(
         for field in fields:
             if '=' in field:
                 dst, src = field.split('=')
-                config.process.split_by_newline[dst] = src
+                #config.process.split_by_newline[dst] = src
+                config.process.split[dst] = SplitConfig(
+                    field = src,
+                    delimiter = '\n',
+                )
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
     if str_filters:
@@ -339,8 +355,8 @@ def convert(
                 new_flat_row = map_constants(new_flat_row, config.process.assign_constants)
             if config.map:
                 new_flat_row = remap_columns(new_flat_row, config.map)
-            if config.process.split_by_newline:
-                new_flat_row = apply_fields_split_by_newline(new_flat_row, config.process.split_by_newline)
+            if config.process.split:
+                new_flat_row = split_fields(new_flat_row, config.process.split)
             if config.process.assign_ids:
                 new_flat_row = assign_id(new_flat_row, config.process.assign_ids, id_context_map)
             if config.process.assign_formats:

python_table_converter-0.2.10/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.10"
-__version_tuple__ = (0, 2, 10)