python-table-converter 0.2.3__tar.gz → 0.2.5__tar.gz

{python_table_converter-0.2.3 → python_table_converter-0.2.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-converter
-Version: 0.2.3
+Version: 0.2.5
 Summary: A table data converter
 Home-page: https://github.com/akivajp/python-table-converter
 License: MIT

{python_table_converter-0.2.3 → python_table_converter-0.2.5}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "python-table-converter"
-version = "0.2.3"
+version = "0.2.5"
 description = "A table data converter"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"

python_table_converter-0.2.5/table_converter/__init__.py

@@ -0,0 +1,2 @@
+__version__ = "0.2.5"
+__version_tuple__ = (0, 2, 5)

python_table_converter-0.2.5/table_converter/core/config.py

@@ -0,0 +1,123 @@
+# -*- coding: utf-8 -*-
+
+from collections import OrderedDict
+import dataclasses
+from typing import Mapping
+
+from icecream import ic
+import yaml
+
+type FlatFieldMap = Mapping[str, str]
+type FieldMap = Mapping[str, str|FieldMap]
+
+@dataclasses.dataclass
+class AssignIdConfig:
+    primary: list[str]
+    #given: list[str] | None = None
+    context: list[str] | None = None
+
+@dataclasses.dataclass
+class ProcessConfig:
+    assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    assign_formats: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    #assign_ids: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+    assign_ids: Mapping[str, AssignIdConfig] = dataclasses.field(default_factory=OrderedDict)
+    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
+
+    def __setitem__(self, key, value):
+        setattr(self, key, value)
+
+@dataclasses.dataclass
+class Config:
+    map: FieldMap = dataclasses.field(default_factory=OrderedDict)
+    process: ProcessConfig = dataclasses.field(default_factory=ProcessConfig)
+
+def flatten(
+    mapping: FieldMap,
+    parent_key: str = '',
+    new_mapping: FlatFieldMap | None = None,
+) -> FlatFieldMap:
+    if new_mapping is None:
+        new_mapping = OrderedDict()
+    for key, mapped in mapping.items():
+        new_key = f'{parent_key}.{key}' if parent_key else key
+        if isinstance(mapped, Mapping):
+            flatten(mapped, new_key, new_mapping)
+        else:
+            new_mapping[new_key] = mapped
+    return new_mapping
+
+def setup_config(
+    config_path: str | None = None,
+):
+    config = Config()
+    if config_path:
+        if config_path.endswith('.yaml'):
+            yaml.add_constructor(
+                yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+                lambda loader, node: OrderedDict(loader.construct_pairs(node)),
+            )
+            with open(config_path, 'r') as f:
+                loaded = yaml.load(f, yaml.Loader)
+        else:
+            raise ValueError(
+                'Only YAML configuration files are supported.'
+            )
+        ic(loaded)
+        if 'map' in loaded:
+            config.map = flatten(loaded['map'])
+        setup_process_config(config, loaded)
+    return config
+
+def setup_process_config(
+    config: Config,
+    loaded: Mapping,
+):
+    dict_process = loaded.get('process')
+    if isinstance(dict_process, Mapping):
+        for process_key in [
+            'assign_constants',
+            'assign_formats',
+            'split_by_newline',
+        ]:
+            dict_subprocess = dict_process.get(process_key)
+            if isinstance(dict_subprocess, Mapping):
+                config.process[process_key] = flatten(loaded['process'][process_key])
+        setup_process_assign_ids_config(config, dict_process)
+
+def setup_process_assign_ids_config(
+    config: Config,
+    dict_process: Mapping,
+):
+    dict_subprocess = dict_process.get('assign_ids')
+    if isinstance(dict_subprocess, Mapping):
+        for key, value in dict_subprocess.items():
+            if isinstance(value, Mapping):
+                primary = value.get('primary')
+                if not primary:
+                    ic.enable()
+                    ic(value)
+                    ic(value.get('primary'))
+                    raise ValueError(
+                        'Primary field is required for assign_ids.'
+                    )
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = value.get('primary', []),
+                    #given = value.get('given', None),
+                    context = value.get('context', None),
+                )
+            elif isinstance(value, list):
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = value,
+                )
+            elif isinstance(value, str):
+                config.process.assign_ids[key] = AssignIdConfig(
+                    primary = [value],
+                )
+            else:
+                ic.enable()
+                ic(value)
+                ic(type(value))
+                raise ValueError(
+                    f'Unsupported assign_ids value type: {type(value)}'
+                )

{python_table_converter-0.2.3 → python_table_converter-0.2.5}/table_converter/core/convert.py

@@ -14,6 +14,13 @@ import pandas as pd
 # local
 
 from . config import setup_config
+from . functions.assign_id import (
+    assign_id,
+    create_id_context_map,
+    setup_assign_ids,
+)
+from . functions.get_field_value import get_field_value
+from . functions.set_field_value import set_field_value
 
 dict_loaders: dict[str, callable] = {}
 def register_loader(
@@ -85,31 +92,6 @@ def save_jsonl(
             )
             f.write('\n')
 
-def set_field_value(
-    data: OrderedDict,
-    field: str,
-    value: any,
-):
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field not in data:
-            data[field] = OrderedDict()
-        set_field_value(data[field], rest, value)
-    else:
-        data[field] = value
-
-def get_field_value(
-    data: OrderedDict,
-    field: str,
-):
-    if field in data:
-        return data[field], True
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field in data:
-            return get_field_value(data[field], rest)
-    return None, False
-
 def search_column_value(
     row: OrderedDict,
     column: str,
@@ -159,7 +141,7 @@ def map_formats(
                 #ic(e.args)
                 #ic(e.args[0])
                 key = e.args[0]
-                params[key] = '__undefined__'
+                params[key] = f'__{key}__undefined__'
             except:
                 raise
         set_field_value(new_row, f'__debug__.{column}', formatted)
@@ -196,46 +178,6 @@ def apply_fields_split_by_newline(
                 set_field_value(new_row, f'__debug__.{column}', value)
     return new_row
 
-def create_id_stat_node():
-    return {
-        'max_id': 0,
-        'dict_value_to_id': {},
-        'dict_id_to_node': {},
-    }
-
-def assign_id_in_node(
-    row: OrderedDict,
-    column: str,
-    dict_assignment: OrderedDict,
-    id_stat_node: dict,
-):
-    value, found = search_column_value(row, dict_assignment[column])
-    if not found:
-        raise KeyError(f'Column not found: {column}, existing columns: {row.keys()}')
-    if value not in id_stat_node['dict_value_to_id']:
-        field_id = id_stat_node['max_id'] + 1
-        id_stat_node['max_id'] = field_id
-        id_stat_node['dict_value_to_id'][value] = field_id
-        node = create_id_stat_node()
-        id_stat_node['dict_id_to_node'][field_id] = node
-    else:
-        field_id = id_stat_node['dict_value_to_id'][value]
-        node = id_stat_node['dict_id_to_node'][field_id]
-    set_field_value(row, f'__debug__.{column}', field_id)
-    set_field_value(row, f'__debug__.__ids__.{column}', field_id)
-    return node
-
-def assign_id(
-    row: OrderedDict,
-    dict_assignment: OrderedDict,
-    root_id_stat_node: dict,
-):
-    new_row = OrderedDict(row)
-    node = root_id_stat_node
-    for column in dict_assignment:
-        node = assign_id_in_node(new_row, column, dict_assignment, node)
-    return new_row
-
 def convert(
     input_files: list[str],
     output_file: str | None = None,
@@ -251,9 +193,7 @@ def convert(
     ic()
     ic(input_files)
     df_list = []
-    dict_formats: OrderedDict | None = None
-    dict_assign_ids= None
-    root_id_stat = create_id_stat_node()
+    id_context_map = create_id_context_map()
     config = setup_config(config_path)
     ic(config)
     if assign_constants:
@@ -265,12 +205,11 @@ def convert(
             else:
                 raise ValueError(f'Invalid constant assignment: {field}')
     if assign_formats:
-        dict_formats = OrderedDict()
         fields = assign_formats.split(',')
         for field in fields:
             if '=' in field:
                 dst, src = field.split('=')
-                dict_formats[dst] = src
+                config.process.assign_formats[dst] = src
             else:
                 raise ValueError(f'Invalid template assignment: {field}')
     if pickup_columns:
@@ -290,20 +229,14 @@ def convert(
             else:
                 raise ValueError(f'Invalid split by newline: {field}')
     if fields_to_assign_ids:
-        dict_assign_ids = OrderedDict()
-        fields = fields_to_assign_ids.split(',')
-        for field in fields:
-            if '=' in field:
-                dst, src = field.split('=')
-                dict_assign_ids[dst] = src
-            else:
-                raise ValueError(f'Invalid id assignment: {field}')
+        setup_assign_ids(config, fields_to_assign_ids)
     if output_file:
         ext = os.path.splitext(output_file)[1]
         if ext not in dict_savers:
             raise ValueError(f'Unsupported file type: {ext}')
         saver = dict_savers[ext]
     ic(config)
+    #return # debug return
     for input_file in input_files:
         ic(input_file)
         if not os.path.exists(input_file):
@@ -331,10 +264,10 @@ def convert(
                 new_row = remap_columns(new_row, config.map)
             if config.process.split_by_newline:
                 new_row = apply_fields_split_by_newline(new_row, config.process.split_by_newline)
-            if dict_assign_ids:
-                new_row = assign_id(new_row, dict_assign_ids, root_id_stat)
-            if dict_formats:
-                new_row = map_formats(new_row, dict_formats)
+            if config.process.assign_ids:
+                new_row = assign_id(new_row, config.process.assign_ids, id_context_map)
+            if config.process.assign_formats:
+                new_row = map_formats(new_row, config.process.assign_formats)
             if config.map:
                 new_row = remap_columns(new_row, config.map)
             if not output_debug:

python_table_converter-0.2.5/table_converter/core/functions/assign_id.py

@@ -0,0 +1,108 @@
+# -*- coding: utf-8 -*-
+
+import dataclasses
+import json
+import os
+
+from collections import OrderedDict
+from collections import defaultdict
+from typing import Mapping
+
+# 3-rd party modules
+
+from icecream import ic
+import numpy as np
+import pandas as pd
+
+# local
+
+from ..config import (
+    AssignIdConfig,
+    Config,
+)
+
+from . search_column_value import search_column_value
+from . set_field_value import set_field_value
+
+type ContextColumnTuple = tuple[str]
+type ContextValueTuple = tuple 
+type PrimaryColumnTuple = tuple[str]
+type PrimaryValueTuple = tuple
+
+@dataclasses.dataclass
+class IdMap:
+    max_id: int = 0
+    dict_value_to_id: Mapping[PrimaryValueTuple, int] = \
+        dataclasses.field(default_factory=defaultdict)
+    dict_id_to_value: Mapping[int, PrimaryValueTuple] = \
+        dataclasses.field(default_factory=defaultdict)
+
+type IdContextMap = Mapping[
+    (
+        ContextColumnTuple,
+        ContextValueTuple,
+        PrimaryColumnTuple,
+    ),
+    IdMap
+]
+
+def create_id_context_map() -> IdContextMap:
+    return defaultdict(IdMap)
+
+def assign_id(
+    row: OrderedDict,
+    dict_assignment: Mapping[str, AssignIdConfig],
+    id_context_map: IdContextMap,
+):
+    new_row = OrderedDict(row)
+    for column, config in dict_assignment.items():
+        context_columns = []
+        context_values = []
+        if config.context:
+            for context_column in config.context:
+                value, found = search_column_value(new_row, context_column)
+                if not found:
+                    raise KeyError(f'Column not found: {context_column}, existing columns: {new_row.keys()}')
+                context_columns.append(context_column)
+                context_values.append(value)
+        primary_columns = []
+        primary_values = []
+        for primary_column in config.primary:
+            value, found = search_column_value(new_row, primary_column)
+            if not found:
+                raise KeyError(f'Column not found: {primary_column}, existing columns: {new_row.keys()}')
+            primary_columns.append(primary_column)
+            primary_values.append(value)
+        context_key = (
+            tuple(context_columns),
+            tuple(context_values),
+            tuple(primary_columns),
+        )
+        primary_value = tuple(primary_values)
+        id_map = id_context_map[context_key]
+        if primary_value not in id_map.dict_value_to_id:
+            field_id = id_map.max_id + 1
+            id_map.max_id = field_id
+            id_map.dict_value_to_id[primary_value] = field_id
+            id_map.dict_id_to_value[field_id] = primary_value
+        else:
+            field_id = id_map.dict_value_to_id[primary_value]
+        set_field_value(new_row, f'__debug__.{column}', field_id)
+    return new_row
+
+def setup_assign_ids(
+    config: Config,
+    fields_to_assign_ids: str,
+):
+    if fields_to_assign_ids:
+        fields = fields_to_assign_ids.split(',')
+        context = []
+        for field in fields:
+            if '=' in field:
+                dst, src = field.split('=')
+                config.process.assign_ids[dst] = AssignIdConfig(
+                    primary = [src],
+                    context = context,
+                )
+            else:
+                raise ValueError(f'Invalid id assignment: {field}')

python_table_converter-0.2.5/table_converter/core/functions/get_field_value.py

@@ -0,0 +1,15 @@
+# Description: Get the value of a field in a dictionary.
+
+from collections import OrderedDict
+
+def get_field_value(
+    data: OrderedDict,
+    field: str,
+):
+    if field in data:
+        return data[field], True
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field in data:
+            return get_field_value(data[field], rest)
+    return None, False

python_table_converter-0.2.5/table_converter/core/functions/search_column_value.py

@@ -0,0 +1,28 @@
+'''
+This function is used to search for a column value in a row. It will first search in the '__debug__' field, then in the '__debug__.__original__' field, and finally in the row itself. If the value is found, it will be set in the row and returned.
+'''
+
+from collections import OrderedDict
+
+from . get_field_value import get_field_value
+from . set_field_value import set_field_value
+
+def search_column_value(
+    row: OrderedDict,
+    column: str,
+):
+    if '__debug__' in row:
+        value, found = get_field_value(row['__debug__'], column)
+        if found:
+            return value, True
+    value, found = get_field_value(row['__debug__'], column)
+    original, found = get_field_value(row, '__debug__.__original__')
+    if found:
+        value, found = get_field_value(original, column)
+        if found:
+            return value, True
+    value, found = get_field_value(row, column)
+    if found:
+        set_field_value(row, column, value)
+        return value, True
+    return None, False

python_table_converter-0.2.5/table_converter/core/functions/set_field_value.py

@@ -0,0 +1,18 @@
+'''
+Set the value of a field in a nested dictionary.
+'''
+
+from collections import OrderedDict
+
+def set_field_value(
+    data: OrderedDict,
+    field: str,
+    value: any,
+):
+    if '.' in field:
+        field, rest = field.split('.', 1)
+        if field not in data:
+            data[field] = OrderedDict()
+        set_field_value(data[field], rest, value)
+    else:
+        data[field] = value

python_table_converter-0.2.3/table_converter/__init__.py

@@ -1,2 +0,0 @@
-__version__ = "0.2.3"
-__version_tuple__ = (0, 2, 3)

python_table_converter-0.2.3/table_converter/core/config.py

@@ -1,70 +0,0 @@
-# -*- coding: utf-8 -*-
-
-from collections import OrderedDict
-import dataclasses
-from typing import Mapping
-
-from icecream import ic
-import yaml
-
-type FlatFieldMap = Mapping[str, str]
-type FieldMap = Mapping[str, str|FieldMap]
-
-@dataclasses.dataclass
-class ProcessConfig:
-    assign_constants: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-    split_by_newline: FlatFieldMap = dataclasses.field(default_factory=OrderedDict)
-
-    def __setitem__(self, key, value):
-        setattr(self, key, value)
-
-@dataclasses.dataclass
-class Config:
-    map: FieldMap = dataclasses.field(default_factory=OrderedDict)
-    process: ProcessConfig = dataclasses.field(default_factory=ProcessConfig)
-
-def flatten(
-    mapping: FieldMap,
-    parent_key: str = '',
-    new_mapping: FlatFieldMap | None = None,
-) -> FlatFieldMap:
-    if new_mapping is None:
-        new_mapping = OrderedDict()
-    for key, mapped in mapping.items():
-        new_key = f'{parent_key}.{key}' if parent_key else key
-        if isinstance(mapped, Mapping):
-            flatten(mapped, new_key, new_mapping)
-        else:
-            new_mapping[new_key] = mapped
-    return new_mapping
-
-def setup_config(
-    config_path: str | None = None,
-):
-    config = Config()
-    if config_path:
-        if config_path.endswith('.yaml'):
-            yaml.add_constructor(
-                yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-                lambda loader, node: OrderedDict(loader.construct_pairs(node)),
-            )
-            with open(config_path, 'r') as f:
-                loaded = yaml.load(f, yaml.Loader)
-        else:
-            raise ValueError(
-                'Only YAML configuration files are supported.'
-            )
-        ic(loaded)
-        if 'map' in loaded:
-            config.map = flatten(loaded['map'])
-        #if 'process' in loaded:
-        dict_process = loaded.get('process')
-        if isinstance(dict_process, Mapping):
-            for process_key in [
-                'assign_constants',
-                'split_by_newline',
-            ]:
-                dict_subprocess = dict_process.get(process_key)
-                if isinstance(dict_subprocess, Mapping):
-                    config.process[process_key] = flatten(loaded['process'][process_key])
-    return config