python-table-processor 0.3.3__py3-none-any.whl → 0.3.5__py3-none-any.whl

{python_table_processor-0.3.3.dist-info → python_table_processor-0.3.5.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-processor
-Version: 0.3.3
+Version: 0.3.5
 Summary: A table data processor
 Home-page: https://github.com/akivajp/python-table-processor
 License: MIT

{python_table_processor-0.3.3.dist-info → python_table_processor-0.3.5.dist-info}/RECORD

@@ -1,23 +1,23 @@
-table_processor/__init__.py,sha256=IqYTIOR96kwEp7nfLi1h2e7pZiFKmLonFJ-wwSegFkw,52
+table_processor/__init__.py,sha256=gtxdmTbOklsCYOkQEth0hoRHsSwKt1ixChUIQPBOStA,52
 table_processor/cli.py,sha256=uluf_VsBzq3Ldxh3ePGseSrv8--lI7huu-6RjDaHPPg,2170
-table_processor/commands/convert_tables.py,sha256=j25KL0zRl5ikrBk73jAkKn4rPmDq7SecChZxecDpILQ,2123
+table_processor/commands/convert_tables.py,sha256=w4IZftSlrbt-BMPjldULT_616LSiNE3h7GmrFF7K5OM,2276
 table_processor/commands/merge_tables.py,sha256=Uu-hjfPh69Of0_OZrY8sY0oscpLSdK4jwJ_t1qgfPMg,1485
-table_processor/core/actions.py,sha256=WLlq-G8Pb9xjsLpXK1QVCwhK20ijz2oIbTFibihdFTE,21557
+table_processor/core/actions.py,sha256=GKUKzoPxUEuWSeuY-HsqFoS5Bdm99bjkRhuBgG1s_uk,23658
 table_processor/core/config.py,sha256=Z1s2r3y4oQXdbDCwIY_hYzoq6VRv7Sb3oUlfBCu13Lc,11646
 table_processor/core/constants.py,sha256=w-1TxtpXywb6Qh8M4YWuzcCHUvCjy5zJYxaXC3RQS6A,158
-table_processor/core/convert.py,sha256=cibzqJ5hHsfX_7qUz4kE2Gi-WH4UMqecBl3kSb4IVsY,11140
+table_processor/core/convert.py,sha256=dzGcpBwJ5omiE8Pw-VmHnGoOqDyh1Rip6BiLdvdf3V8,11561
 table_processor/core/functions/assign_id.py,sha256=dA-wgZA7E01gbx8qGvo1PeMd3aq9PhMdyITzr3BKXIE,4204
 table_processor/core/functions/flatten_row.py,sha256=2l-s4YXgBM58IthpfVzd7L4EGUCekSi7ss5CaM_3MUU,686
-table_processor/core/functions/get_nested_field_value.py,sha256=vF4pH6nxR3-2bDfAvrMfHcPFNiS4MnGkXL8FdcScdIw,685
+table_processor/core/functions/get_nested_field_value.py,sha256=ZEJi8XtDONoYmhPoqYpVZRGtEWvsfAksV4FXtkMAaq8,946
 table_processor/core/functions/nest_row.py,sha256=b5AKfxE39g-NOLranQRy0dfTZq-27Or8ZlgCB8N5kFs,640
 table_processor/core/functions/search_column_value.py,sha256=GscwLaMTORHFDhdCdjzdoy46UGyFHrGwDQA2b_tZhsQ,740
 table_processor/core/functions/set_flat_field_value.py,sha256=7-BgD2yEUGyBaFaIJ18bYv82Qdy0CN8NiC5ZtsAdv44,531
 table_processor/core/functions/set_nested_field_value.py,sha256=csO0v_nW94q-M9bNL91xR-TLHPXDDxnLK8gN-0LLd6k,595
 table_processor/core/functions/set_row_value.py,sha256=wlF_nVGZY74XCzd6pBJJyti62m6y0s1kc5Q6uq9jn8o,641
-table_processor/core/merge.py,sha256=Yc5OV9n1xV9QtgY4srl6Susd5cOKbuYseEtYuBampDw,4672
-table_processor/core/types.py,sha256=qASw6zDJXFpaswKjhSQ39wcTH8BcCmp7xKW_hgP-RRY,2665
-python_table_processor-0.3.3.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
-python_table_processor-0.3.3.dist-info/METADATA,sha256=mvhOnT4vISkKBt5HCEecLCTIRrwrWivJyn63II1w5Jk,1098
-python_table_processor-0.3.3.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-python_table_processor-0.3.3.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
-python_table_processor-0.3.3.dist-info/RECORD,,
+table_processor/core/merge.py,sha256=NoxVfhpOr7oxfmMZt1Xqx94fkIhAJQcYuYzR3XcJsns,4809
+table_processor/core/types.py,sha256=xNvy1vL1ei87kt-MgSfRJSllYrNKRo23pHtJA15DyLQ,2895
+python_table_processor-0.3.5.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
+python_table_processor-0.3.5.dist-info/METADATA,sha256=MA1YvRPDWpWhbua0uFd5AOyFJ7EtxUYJ9XV3bYyr2S8,1098
+python_table_processor-0.3.5.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+python_table_processor-0.3.5.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
+python_table_processor-0.3.5.dist-info/RECORD,,

table_processor/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.3.3"
-__version_tuple__ = (0, 3, 3)
+__version__ = "0.3.5"
+__version_tuple__ = (0, 3, 5)

table_processor/commands/convert_tables.py

@@ -20,6 +20,7 @@ def run(
         output_debug = args.output_debug,
         verbose = args.verbose,
         ignore_file_rows = args.ignore_file_rows,
+        skip_header = args.skip_header,
     )
 
 def setup_parser(
@@ -77,4 +78,9 @@ def setup_parser(
         action='store_true',
         help='Output debug information',
     )
+    parser.add_argument(
+        '--skip-header',
+        action='store_true',
+        help='Skip header',
+    )
     parser.set_defaults(handler=run)

table_processor/core/actions.py

@@ -29,6 +29,7 @@ from . types import (
     AssignFormatConfig,
     AssignIdConfig,
     AssignLengthConfig,
+    CastConfig,
     FilterConfig,
     GlobalStatus,
     JoinConfig,
@@ -147,6 +148,31 @@ def setup_actions_with_args(
                     source = source,
                 ))
                 continue
+            if action_name == 'cast':
+                required = options.get('required', False)
+                as_type = options.get('as', 'literal')
+                if as_type in ['boolean']:
+                    as_type = 'bool'
+                if as_type not in ['bool', 'int', 'float', 'str']:
+                    raise ValueError(
+                        f'Unsupported as type: {as_type}'
+                    )
+                assign_default = False
+                default_value = None
+                if 'default' in options:
+                    assign_default = True
+                    default_value = options['default']
+                    if default_value in ['None', 'none', 'Null', 'null']:
+                        default_value = None
+                config.actions.append(CastConfig(
+                    target = target,
+                    source = source,
+                    as_type = as_type,
+                    required = required,
+                    assign_default = assign_default,
+                    default_value = default_value,
+                ))
+                continue
             if action_name == 'filter-empty':
                 config.actions.append(FilterConfig(
                     field = target,
@@ -323,6 +349,8 @@ def do_action(
         return assign_id(status.id_context_map, row, action)
     if isinstance(action, AssignLengthConfig):
         return assign_length(row, action)
+    if isinstance(action, CastConfig):
+        return cast(row, action)
     if isinstance(action, FilterConfig):
         if filter_row(row, action):
             return row
@@ -685,3 +713,37 @@ def assign_length(
     if found:
         set_row_staging_value(row, config.target, len(value))
     return row
+
+def cast(
+    row: Row,
+    config: CastConfig,
+):
+    value, found = search_column_value(row.nested, config.source)
+    if config.required:
+        if not found:
+            raise ValueError(
+                f'Required field not found, field: {config.source}'
+            )
+    if config.as_type == 'bool':
+        cast_func = bool
+    elif config.as_type == 'int':
+        cast_func = int
+    elif config.as_type == 'float':
+        cast_func = float
+    elif config.as_type == 'str':
+        cast_func = str
+    else:
+        raise ValueError(
+            f'Unsupported as type: {config.as_type}'
+        )
+    try:
+        casted = cast_func(value)
+    except:
+        if config.assign_default:
+            casted = config.default_value
+        else:
+            raise ValueError(
+                f'Failed to cast: {value}'
+            )
+    set_row_staging_value(row, config.target, casted)
+    return row

table_processor/core/convert.py

@@ -64,12 +64,13 @@ def register_loader(
 
 def load(
     input_file: str,
+    **kwargs,
 ):
     ext = os.path.splitext(input_file)[1]
     if ext not in dict_loaders:
         raise ValueError(f'Unsupported file type: {ext}')
     loader = dict_loaders[ext]
-    return loader(input_file)
+    return loader(input_file, **kwargs)
 
 dict_savers: dict[str, callable] = {}
 def register_saver(
@@ -93,16 +94,34 @@ def save(
 @register_loader('.csv')
 def load_csv(
     input_file: str,
+    **kwargs,
 ):
+    skip_header = kwargs.get('skip_header', False)
     # utf-8
     #df = pd.read_csv(input_file)
     # UTF-8 with BOM
-    df = pd.read_csv(input_file, encoding='utf-8-sig')
+    if skip_header:
+        df = pd.read_csv(
+            input_file,
+            encoding='utf-8-sig',
+            header=None,
+        )
+        #new_column_names = [f'__values__.{i}' for i in df.columns]
+        new_column_names = [f'{i}' for i in df.columns]
+        df = df.rename(columns=dict(
+            zip(df.columns, new_column_names)
+        ))
+    else:
+        df = pd.read_csv(
+            input_file,
+            encoding='utf-8-sig',
+        )
     return df
 
 @register_loader('.xlsx')
 def load_excel(
     input_file: str,
+    **kwargs,
 ):
     #df = pd.read_excel(input_file)
     # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
@@ -123,6 +142,7 @@ def load_excel(
 @register_loader('.json')
 def load_json(
     input_file: str,
+    **kiwargs,
 ):
     with open(input_file, 'r') as f:
         data = json.load(f)
@@ -165,6 +185,7 @@ def save_json(
 @register_loader('.jsonl')
 def load_jsonl(
     input_file: str,
+    **kwargs,
 ):
     rows = []
     with open(input_file, 'r') as f:
@@ -274,6 +295,7 @@ def convert(
     action_delimiter: str = ':',
     verbose: bool = False,
     ignore_file_rows: list[str] | None = None,
+    skip_header: bool = False,
 ):
     ic.enable()
     ic()
@@ -306,11 +328,7 @@ def convert(
         if not os.path.exists(input_file):
             raise FileNotFoundError(f'File not found: {input_file}')
         base_name = os.path.basename(input_file)
-        ext = os.path.splitext(input_file)[1]
-        ic(ext)
-        if ext not in dict_loaders:
-            raise ValueError(f'Unsupported file type: {ext}')
-        df = dict_loaders[ext](input_file)
+        df = load(input_file, skip_header=skip_header)
         # NOTE: NaN を None に変換しておかないと厄介
         df = df.replace([np.nan], [None])
         #ic(df)

table_processor/core/functions/get_nested_field_value.py

@@ -15,11 +15,17 @@ def get_nested_field_value(
             if index < len(data):
                 return data[index], True
             return None, False
+        if '.' in field:
+            field, rest = field.split('.', 1)
+            if field.isdigit():
+                index = int(field)
+                if index < len(data):
+                    return get_nested_field_value(data[index], rest)
     if isinstance(data, dict):
         if field in data:
             return data[field], True
-    if '.' in field:
-        field, rest = field.split('.', 1)
-        if field in data:
-            return get_nested_field_value(data[field], rest)
+        if '.' in field:
+            field, rest = field.split('.', 1)
+            if field in data:
+                return get_nested_field_value(data[field], rest)
     return None, False

table_processor/core/merge.py

@@ -116,17 +116,20 @@ def merge(
             #    dict_key_to_row[key].flat.update(row.flat)
             if primary_key not in dict_key_to_row:
                 if ignore_not_found:
+                    ic(primary_key)
+                    ic(row.flat['__staging__.__file_row_index__'])
                     list_ignored_keys.append(primary_key)
                     continue
                 ic(index)
-                raise ValueError(f'Key not found: {key}')
+                raise ValueError(f'Key not found: {primary_key}')
             previous_row = dict_key_to_row[primary_key]
             #ic(previous_row)
             #ic(previous_row.flat)
             for key, value in row.flat.items():
                 if key.startswith('__staging__.'):
                     continue
-                ic(key, value)
+                #ic(key)
+                #ic(key, value)
                 set_row_value(previous_row, key, value)
                 #set_row_value(previous_row, '指示追従性？', 'test')
                 #set_row_value(previous_row, 'modified', True)

table_processor/core/types.py

@@ -51,6 +51,17 @@ class AssignLengthConfig:
     target: str
     source: str
 
+@dataclasses.dataclass
+class CastConfig:
+    target: str
+    source: str
+    as_type: Literal[
+        'bool', 'int', 'float', 'str'
+    ]
+    required: bool = False
+    assign_default: bool = False
+    default_value: Any = None
+
 @dataclasses.dataclass
 class FilterConfig:
     field: str