python-table-converter 0.2.4__tar.gz → 0.2.6__tar.gz

{python_table_converter-0.2.4 → python_table_converter-0.2.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.4
+Version: 0.2.6
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.4 → python_table_converter-0.2.6}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.4"
+version = "0.2.6"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.6/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.6"
+__version_tuple__ = (0, 2, 6)

python_table_converter-0.2.6/table_converter/core/config.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+
+from collections import OrderedDict
+import dataclasses
+from typing import Mapping
+
+from icecream import ic
+import yaml
+
+type FlatFieldMap = Mapping[str, str]
+type FieldMap = Mapping[str, str|FieldMap]
+
+@dataclasses.dataclass
+class AssignIdConfig:
+    primary: list[str]
+    #given: list[str] | None = None
+    context: list[str] | None = None
+
+@dataclasses.dataclass
+class ProcessConfig:
+    assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    #assign_ids: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    assign_ids: Mapping[str, AssignIdConfig] = dataclasses.field(default_factory=OrderedDict)
+    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+@dataclasses.dataclass
+class Config:
+    map: FieldMap = dataclasses.field(default_factory=OrderedDict)
+    process: ProcessConfig = dataclasses.field(default_factory=ProcessConfig)
+
+def flatten(
+    mapping: FieldMap,
+    parent_key: str = '',
+    new_mapping: FlatFieldMap | None = None,
+) -> FlatFieldMap:
+    if new_mapping is None:
+        new_mapping = OrderedDict()
+    for key, mapped in mapping.items():
+        new_key = f'{parent_key}.{key}' if parent_key else key
+        if isinstance(mapped, Mapping):
+            flatten(mapped, new_key, new_mapping)
+        else:
+            new_mapping[new_key] = mapped
+    return new_mapping
+
+def setup_config(
+    config_path: str | None = None,
+):
+    config = Config()
+    if config_path:
+        if config_path.endswith('.yaml'):
+            yaml.add_constructor(
+                yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                lambda loader, node: OrderedDict(loader.construct_pairs(node)),
+            )
+            with open(config_path, 'r') as f:
+                loaded = yaml.load(f, yaml.Loader)
+        else:
+            raise ValueError(
+                'Only YAML configuration files are supported.'
+            )
+        ic(loaded)
+        if 'map' in loaded:
+            config.map = flatten(loaded['map'])
+        setup_process_config(config, loaded)
+    return config
+
+def setup_process_config(
+    config: Config,
+    loaded: Mapping,
+):
+    dict_process = loaded.get('process')
+    if isinstance(dict_process, Mapping):
+        for process_key in [
+            'assign_constants',
+            'assign_formats',
+            'split_by_newline',
+        ]:
+            dict_subprocess = dict_process.get(process_key)
+            if isinstance(dict_subprocess, Mapping):
+                config.process[process_key] = flatten(loaded['process'][process_key])
+        setup_process_assign_ids_config(config, dict_process)
+
+def setup_process_assign_ids_config(
+    config: Config,
+    dict_process: Mapping,
+):
+    dict_subprocess = dict_process.get('assign_ids')
+    if isinstance(dict_subprocess, Mapping):
+        for key, value in dict_subprocess.items():
+            if isinstance(value, Mapping):
+                primary = value.get('primary')
+                if not primary:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('primary'))
+                    raise ValueError(
+                        'Primary field is required for assign_ids.'
+                    )
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = value.get('primary', []),
+                    #given = value.get('given', None),
+                    context = value.get('context', None),
+                )
+            elif isinstance(value, list):
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = value,
+                )
+            elif isinstance(value, str):
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = [value],
+                )
+            else:
+                ic.enable()
+                ic(value)
+                ic(type(value))
+                raise ValueError(
+                    f'Unsupported assign_ids value type: {type(value)}'
+                )

python_table_converter-0.2.6/table_converter/core/constants.py

@@ -0,0 +1,3 @@
+FILE_FIELD = '__file__'
+INPUT_FIELD = '__input__'
+STAGING_FIELD = '__staging__'

{python_table_converter-0.2.4 → python_table_converter-0.2.6}/table_converter/core/convert.py

@@ -14,6 +14,18 @@ import pandas as pd
 # local
 
 from . config import setup_config
+from . constants import (
+    FILE_FIELD,
+    INPUT_FIELD,
+    STAGING_FIELD,
+)
+from . functions.assign_id import (
+    assign_id,
+    create_id_context_map,
+    setup_assign_ids,
+)
+from . functions.get_field_value import get_field_value
+from . functions.set_field_value import set_field_value
 
 dict_loaders: dict[str, callable] = {}
 def register_loader(
@@ -85,41 +97,16 @@ def save_jsonl(
             )
             f.write('\n')
 
-def set_field_value(
-    data: OrderedDict,
-    field: str,
-    value: any,
-):
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field not in data:
-            data[field] = OrderedDict()
-        set_field_value(data[field], rest, value)
-    else:
-        data[field] = value
-
-def get_field_value(
-    data: OrderedDict,
-    field: str,
-):
-    if field in data:
-        return data[field], True
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field in data:
-            return get_field_value(data[field], rest)
-    return None, False
-
 def search_column_value(
     row: OrderedDict,
     column: str,
 ):
-    if '__debug__' in row:
-        value, found = get_field_value(row['__debug__'], column)
+    if STAGING_FIELD in row:
+        value, found = get_field_value(row[STAGING_FIELD], column)
         if found:
             return value, True
-    value, found = get_field_value(row['__debug__'], column)
-    original, found = get_field_value(row, '__debug__.__original__')
+    value, found = get_field_value(row[STAGING_FIELD], column)
+    original, found = get_field_value(row, f'{STAGING_FIELD}.{INPUT_FIELD}')
     if found:
         value, found = get_field_value(original, column)
         if found:
@@ -137,7 +124,7 @@ def map_constants(
     new_row = OrderedDict(row)
     for column in dict_constants.keys():
         #set_field_value(new_row, column, dict_constants[column])
-        set_field_value(new_row, f'__debug__.{column}', dict_constants[column])
+        set_field_value(new_row, f'{STAGING_FIELD}.{column}', dict_constants[column])
     return new_row
 
 def map_formats(
@@ -148,7 +135,7 @@ def map_formats(
     for column in dict_formats.keys():
         template = dict_formats[column]
         params = {}
-        params.update(row['__debug__'])
+        params.update(row[STAGING_FIELD])
         params.update(row)
         formatted = None
         while formatted is None:
@@ -159,10 +146,10 @@ def map_formats(
                 #ic(e.args)
                 #ic(e.args[0])
                 key = e.args[0]
-                params[key] = '__undefined__'
+                params[key] = f'__{key}__undefined__'
             except:
                 raise
-        set_field_value(new_row, f'__debug__.{column}', formatted)
+        set_field_value(new_row, f'{STAGING_FIELD}.{column}', formatted)
     return new_row
 
 def remap_columns(
@@ -175,7 +162,7 @@ def remap_columns(
         if found:
             set_field_value(new_row, column, value)
     for column in row.keys():
-        if column == '__debug__':
+        if column == STAGING_FIELD:
             # NOTE: Ignore debug fields
             set_field_value(new_row, column, row[column])
     return new_row
@@ -191,49 +178,9 @@ def apply_fields_split_by_newline(
         if found:
             if isinstance(value, str):
                 new_value = value.split('\n')
-                set_field_value(new_row, f'__debug__.{column}', new_value)
+                set_field_value(new_row, f'{STAGING_FIELD}.{column}', new_value)
             else:
-                set_field_value(new_row, f'__debug__.{column}', value)
-    return new_row
-
-def create_id_stat_node():
-    return {
-        'max_id': 0,
-        'dict_value_to_id': {},
-        'dict_id_to_node': {},
-    }
-
-def assign_id_in_node(
-    row: OrderedDict,
-    column: str,
-    dict_assignment: OrderedDict,
-    id_stat_node: dict,
-):
-    value, found = search_column_value(row, dict_assignment[column])
-    if not found:
-        raise KeyError(f'Column not found: {column}, existing columns: {row.keys()}')
-    if value not in id_stat_node['dict_value_to_id']:
-        field_id = id_stat_node['max_id'] + 1
-        id_stat_node['max_id'] = field_id
-        id_stat_node['dict_value_to_id'][value] = field_id
-        node = create_id_stat_node()
-        id_stat_node['dict_id_to_node'][field_id] = node
-    else:
-        field_id = id_stat_node['dict_value_to_id'][value]
-        node = id_stat_node['dict_id_to_node'][field_id]
-    set_field_value(row, f'__debug__.{column}', field_id)
-    set_field_value(row, f'__debug__.__ids__.{column}', field_id)
-    return node
-
-def assign_id(
-    row: OrderedDict,
-    dict_assignment: OrderedDict,
-    root_id_stat_node: dict,
-):
-    new_row = OrderedDict(row)
-    node = root_id_stat_node
-    for column in dict_assignment:
-        node = assign_id_in_node(new_row, column, dict_assignment, node)
+                set_field_value(new_row, f'{STAGING_FIELD}.{column}', value)
     return new_row
 
 def convert(
@@ -251,8 +198,7 @@ def convert(
     ic()
     ic(input_files)
     df_list = []
-    dict_assign_ids= None
-    root_id_stat = create_id_stat_node()
+    id_context_map = create_id_context_map()
     config = setup_config(config_path)
     ic(config)
     if assign_constants:
@@ -288,14 +234,7 @@ def convert(
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
     if fields_to_assign_ids:
-        dict_assign_ids = OrderedDict()
-        fields = fields_to_assign_ids.split(',')
-        for field in fields:
-            if '=' in field:
-                dst, src = field.split('=')
-                dict_assign_ids[dst] = src
-            else:
-                raise ValueError(f'Invalid id assignment: {field}')
+        setup_assign_ids(config, fields_to_assign_ids)
     if output_file:
         ext = os.path.splitext(output_file)[1]
         if ext not in dict_savers:
@@ -322,22 +261,22 @@ def convert(
         for index, row in df.iterrows():
             orig = OrderedDict(row)
             new_row = OrderedDict(row)
-            set_field_value(new_row, '__debug__.__original__', orig)
-            set_field_value(new_row, '__debug__.__file__', input_file)
+            set_field_value(new_row, f'{STAGING_FIELD}.{INPUT_FIELD}', orig)
+            set_field_value(new_row, f'{STAGING_FIELD}.{FILE_FIELD}', input_file)
             if config.process.assign_constants:
                 new_row = map_constants(new_row, config.process.assign_constants)
             if config.map:
                 new_row = remap_columns(new_row, config.map)
             if config.process.split_by_newline:
                 new_row = apply_fields_split_by_newline(new_row, config.process.split_by_newline)
-            if dict_assign_ids:
-                new_row = assign_id(new_row, dict_assign_ids, root_id_stat)
+            if config.process.assign_ids:
+                new_row = assign_id(new_row, config.process.assign_ids, id_context_map)
             if config.process.assign_formats:
                 new_row = map_formats(new_row, config.process.assign_formats)
             if config.map:
                 new_row = remap_columns(new_row, config.map)
             if not output_debug:
-                new_row.pop('__debug__', None)
+                new_row.pop(STAGING_FIELD, None)
             new_rows.append(new_row)
         new_df = pd.DataFrame(new_rows)
         df_list.append(new_df)

python_table_converter-0.2.6/table_converter/core/functions/assign_id.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+import dataclasses
+import json
+import os
+
+from collections import OrderedDict
+from collections import defaultdict
+from typing import Mapping
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+
+# local
+
+from ..config import (
+    AssignIdConfig,
+    Config,
+)
+
+from . search_column_value import search_column_value
+from . set_field_value import set_field_value
+
+type ContextColumnTuple = tuple[str]
+type ContextValueTuple = tuple 
+type PrimaryColumnTuple = tuple[str]
+type PrimaryValueTuple = tuple
+
+@dataclasses.dataclass
+class IdMap:
+    max_id: int = 0
+    dict_value_to_id: Mapping[PrimaryValueTuple, int] = \
+        dataclasses.field(default_factory=defaultdict)
+    dict_id_to_value: Mapping[int, PrimaryValueTuple] = \
+        dataclasses.field(default_factory=defaultdict)
+
+type IdContextMap = Mapping[
+    (
+        ContextColumnTuple,
+        ContextValueTuple,
+        PrimaryColumnTuple,
+    ),
+    IdMap
+]
+
+def create_id_context_map() -> IdContextMap:
+    return defaultdict(IdMap)
+
+def assign_id(
+    row: OrderedDict,
+    dict_assignment: Mapping[str, AssignIdConfig],
+    id_context_map: IdContextMap,
+):
+    new_row = OrderedDict(row)
+    for column, config in dict_assignment.items():
+        context_columns = []
+        context_values = []
+        if config.context:
+            for context_column in config.context:
+                value, found = search_column_value(new_row, context_column)
+                if not found:
+                    raise KeyError(f'Column not found: {context_column}, existing columns: {new_row.keys()}')
+                context_columns.append(context_column)
+                context_values.append(value)
+        primary_columns = []
+        primary_values = []
+        for primary_column in config.primary:
+            value, found = search_column_value(new_row, primary_column)
+            if not found:
+                raise KeyError(f'Column not found: {primary_column}, existing columns: {new_row.keys()}')
+            primary_columns.append(primary_column)
+            primary_values.append(value)
+        context_key = (
+            tuple(context_columns),
+            tuple(context_values),
+            tuple(primary_columns),
+        )
+        primary_value = tuple(primary_values)
+        id_map = id_context_map[context_key]
+        if primary_value not in id_map.dict_value_to_id:
+            field_id = id_map.max_id + 1
+            id_map.max_id = field_id
+            id_map.dict_value_to_id[primary_value] = field_id
+            id_map.dict_id_to_value[field_id] = primary_value
+        else:
+            field_id = id_map.dict_value_to_id[primary_value]
+        set_field_value(new_row, f'__debug__.{column}', field_id)
+    return new_row
+
+def setup_assign_ids(
+    config: Config,
+    fields_to_assign_ids: str,
+):
+    if fields_to_assign_ids:
+        fields = fields_to_assign_ids.split(',')
+        context = []
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                config.process.assign_ids[dst] = AssignIdConfig(
+                    primary = [src],
+                    context = context,
+                )
+            else:
+                raise ValueError(f'Invalid id assignment: {field}')

python_table_converter-0.2.6/table_converter/core/functions/get_field_value.py

@@ -0,0 +1,15 @@
+# Description: Get the value of a field in a dictionary.
+
+from collections import OrderedDict
+
+def get_field_value(
+    data: OrderedDict,
+    field: str,
+):
+    if field in data:
+        return data[field], True
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field in data:
+            return get_field_value(data[field], rest)
+    return None, False

python_table_converter-0.2.6/table_converter/core/functions/search_column_value.py

@@ -0,0 +1,30 @@
+'''
+This function is used to search for a column value in a row. It will first search in the '__debug__' field, then in the '__debug__.__original__' field, and finally in the row itself. If the value is found, it will be set in the row and returned.
+'''
+
+from .. constants import STAGING_FIELD
+
+from collections import OrderedDict
+
+from . get_field_value import get_field_value
+from . set_field_value import set_field_value
+
+def search_column_value(
+    row: OrderedDict,
+    column: str,
+):
+    if STAGING_FIELD in row:
+        value, found = get_field_value(row[STAGING_FIELD], column)
+        if found:
+            return value, True
+    value, found = get_field_value(row[STAGING_FIELD], column)
+    original, found = get_field_value(row, f'{STAGING_FIELD}.__original__')
+    if found:
+        value, found = get_field_value(original, column)
+        if found:
+            return value, True
+    value, found = get_field_value(row, column)
+    if found:
+        set_field_value(row, column, value)
+        return value, True
+    return None, False

python_table_converter-0.2.6/table_converter/core/functions/set_field_value.py

@@ -0,0 +1,18 @@
+'''
+Set the value of a field in a nested dictionary.
+'''
+
+from collections import OrderedDict
+
+def set_field_value(
+    data: OrderedDict,
+    field: str,
+    value: any,
+):
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field not in data:
+            data[field] = OrderedDict()
+        set_field_value(data[field], rest, value)
+    else:
+        data[field] = value

python_table_converter-0.2.4/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.4"
-__version_tuple__ = (0, 2, 4)

python_table_converter-0.2.4/table_converter/core/config.py

@@ -1,71 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from collections import OrderedDict
-import dataclasses
-from typing import Mapping
-
-from icecream import ic
-import yaml
-
-type FlatFieldMap = Mapping[str, str]
-type FieldMap = Mapping[str, str|FieldMap]
-
-@dataclasses.dataclass
-class ProcessConfig:
-    assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-@dataclasses.dataclass
-class Config:
-    map: FieldMap = dataclasses.field(default_factory=OrderedDict)
-    process: ProcessConfig = dataclasses.field(default_factory=ProcessConfig)
-
-def flatten(
-    mapping: FieldMap,
-    parent_key: str = '',
-    new_mapping: FlatFieldMap | None = None,
-) -> FlatFieldMap:
-    if new_mapping is None:
-        new_mapping = OrderedDict()
-    for key, mapped in mapping.items():
-        new_key = f'{parent_key}.{key}' if parent_key else key
-        if isinstance(mapped, Mapping):
-            flatten(mapped, new_key, new_mapping)
-        else:
-            new_mapping[new_key] = mapped
-    return new_mapping
-
-def setup_config(
-    config_path: str | None = None,
-):
-    config = Config()
-    if config_path:
-        if config_path.endswith('.yaml'):
-            yaml.add_constructor(
-                yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-                lambda loader, node: OrderedDict(loader.construct_pairs(node)),
-            )
-            with open(config_path, 'r') as f:
-                loaded = yaml.load(f, yaml.Loader)
-        else:
-            raise ValueError(
-                'Only YAML configuration files are supported.'
-            )
-        ic(loaded)
-        if 'map' in loaded:
-            config.map = flatten(loaded['map'])
-        dict_process = loaded.get('process')
-        if isinstance(dict_process, Mapping):
-            for process_key in [
-                'assign_constants',
-                'assign_formats',
-                'split_by_newline',
-            ]:
-                dict_subprocess = dict_process.get(process_key)
-                if isinstance(dict_subprocess, Mapping):
-                    config.process[process_key] = flatten(loaded['process'][process_key])
-    return config