tabpro 0.4.2__tar.gz → 0.4.4__tar.gz

{tabpro-0.4.2 → tabpro-0.4.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: tabpro
-Version: 0.4.2
+Version: 0.4.4
 Summary: A table data processor
 Home-page: https://github.com/akivajp/tabpro
 License: MIT
@@ -16,7 +16,9 @@ Requires-Dist: icecream (>=2.1.3,<3.0.0)
 Requires-Dist: logzero (>=1.7.0,<2.0.0)
 Requires-Dist: openpyxl (>=3.1.5,<4.0.0)
 Requires-Dist: pandas (>=2.2.3,<3.0.0)
+Requires-Dist: pytest (>=8.3.5,<9.0.0)
 Requires-Dist: pyyaml (>=6.0.2,<7.0.0)
+Requires-Dist: rich (>=13.9.4,<14.0.0)
 Requires-Dist: tqdm (>=4.67.0,<5.0.0)
 Requires-Dist: xlsxwriter (>=3.2.0,<4.0.0)
 Project-URL: Repository, https://github.com/akivajp/tabpro

{tabpro-0.4.2 → tabpro-0.4.4}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "tabpro"
-version = "0.4.2"
+version = "0.4.4"
 description = "A table data processor"
 authors = ["Akiva Miura <akiva.miura@gmail.com>"]
 license = "MIT"
@@ -30,6 +30,8 @@ pandas = "^2.2.3"
 openpyxl = "^3.1.5"
 pyyaml = "^6.0.2"
 xlsxwriter = "^3.2.0"
+pytest = "^8.3.5"
+rich = "^13.9.4"
 
 
 [build-system]

tabpro-0.4.4/tabpro/__init__.py

@@ -0,0 +1,14 @@
+__version__ = "0.4.4"
+__version_tuple__ = (0, 4, 4)
+
+from . core.io import (
+    get_loader,
+    get_writer,
+    save,
+)
+
+__all__ = [
+    'get_loader',
+    'get_writer',
+    'save',
+]

{tabpro-0.4.2 → tabpro-0.4.4}/tabpro/commands/convert_tables.py

@@ -20,7 +20,7 @@ def run(
         output_debug = args.output_debug,
         verbose = args.verbose,
         ignore_file_rows = args.ignore_file_rows,
-        skip_header = args.skip_header,
+        no_header = args.no_header,
     )
 
 def setup_parser(
@@ -79,8 +79,8 @@ def setup_parser(
         help='Output debug information',
     )
     parser.add_argument(
-        '--skip-header',
+        '--no-header',
         action='store_true',
-        help='Skip header',
+        help='CSV/TSV like data without header row',
     )
     parser.set_defaults(handler=run)

{tabpro-0.4.2 → tabpro-0.4.4}/tabpro/core/actions.py

@@ -38,9 +38,11 @@ from . types import (
     PickConfig,
     PushConfig,
     SplitConfig,
-    Row,
+    #Row,
 )
 
+from . classes.row import Row
+
 from . functions.assign_id import assign_id
 from . functions.flatten_row import flatten_row
 from . functions.nest_row import nest_row

tabpro-0.4.4/tabpro/core/classes/row.py

@@ -0,0 +1,83 @@
+'''
+Row class
+'''
+
+from collections import (
+    OrderedDict
+)
+
+from typing import (
+    Mapping,
+)
+
+from .. functions.get_nested_field_value import get_nested_field_value
+from .. functions.set_nested_field_value import set_nested_field_value
+from .. functions.set_flat_field_value import set_flat_field_value
+
+class Row(Mapping):
+    def __init__(
+            self,
+        ):
+        self.flat = OrderedDict()
+        self.nested = OrderedDict()
+        self._prefix: str | None = None
+        self._staging: Row | None = None
+
+    @property
+    def staging(self):
+        if self._staging is None:
+            row = Row()
+            row.flat = self.flat
+            row.nested = self.nested
+            row._prefix = '__staging__'
+            self._staging = row
+        return self._staging
+
+    def clone(self):
+        cloned = self.__class__.from_dict(self.flat)
+        cloned._prefix = self._prefix
+        return cloned
+
+    def get(self, key, default=None):
+        if self._prefix:
+            key = f'{self._prefix}.{key}'
+        value, found = get_nested_field_value(self.nested, key)
+        if not found:
+            return default
+        return value
+
+    def __getitem__(self, key):
+        if self._prefix:
+            key = f'{self._prefix}.{key}'
+        value, found = get_nested_field_value(self.nested, key)
+        if not found:
+            raise KeyError(f'Key not found: {key}')
+        return value
+
+    def __setitem__(self, key, value):
+        if self._prefix:
+            key = f'{self._prefix}.{key}'
+        set_nested_field_value(self.nested, key, value)
+        set_flat_field_value(self.flat, key, value)
+
+    def __contains__(self, key):
+        if self._prefix:
+            key = f'{self._prefix}.{key}'
+        _, found = get_nested_field_value(self.nested, key)
+        return found
+
+    def __iter__(self):
+        return iter(self.flat)
+
+    def __len__(self):
+        return len(self.flat)
+
+    def __repr__(self):
+        return f'Row(flat={self.flat}, nested={self.nested})'
+    
+    @staticmethod
+    def from_dict(data: dict):
+        row = Row()
+        for key, value in data.items():
+            row[key] = value
+        return row

{tabpro-0.4.2 → tabpro-0.4.4}/tabpro/core/convert.py

@@ -1,11 +1,16 @@
 # -*- coding: utf-8 -*-
 
 import json
-import math
 import os
+import sys
 
 from collections import OrderedDict
 
+import rich
+from rich.console import Console
+from rich.panel import Panel
+from rich.text import Text
+
 from typing import (
     Mapping,
 )
@@ -13,9 +18,12 @@ from typing import (
 # 3-rd party modules
 
 from icecream import ic
+from logzero import logger
 import numpy as np
 import pandas as pd
 
+from . progress import Progress
+
 # local
 
 from . config import (
@@ -33,11 +41,6 @@ from . constants import (
 from . functions.get_nested_field_value import get_nested_field_value
 from . functions.get_nested_field_value import get_nested_field_value
 from . functions.search_column_value import search_column_value
-from . functions.set_nested_field_value import set_nested_field_value
-from . functions.set_row_value import (
-    set_row_value,
-    set_row_staging_value,
-)
 
 from . actions import (
     do_actions,
@@ -53,11 +56,22 @@ from . types import (
 
 from . io import (
     get_loader,
-    get_saver,
-    load,
-    save,
+    #get_saver,
+    get_writer,
+    #load,
+    #save,
 )
 
+def capture_dict(
+    row: dict,
+):
+    console = Console()
+    with console.capture() as capture:
+        #console.print(row)
+        console.print_json(data=row)
+    text = Text.from_ansi(capture.get())
+    return text
+
 def assign_array(
     row: OrderedDict,
     dict_config: Mapping[str, list[AssignArrayConfig]],
@@ -106,17 +120,22 @@ def convert(
     action_delimiter: str = ':',
     verbose: bool = False,
     ignore_file_rows: list[str] | None = None,
-    skip_header: bool = False,
+    no_header: bool = False,
 ):
-    ic.enable()
-    ic()
-    ic(input_files)
-    df_list = []
+    #console = Console()
+    progress = Progress(
+        #console = console,
+        redirect_stdout = False,
+    )
+    progress.start()
+    #ic.enable()
+    console = progress.console
+    console.log('input_files: ', input_files)
     row_list_filtered_out = []
     set_ignore_file_rows = set()
     global_status = GlobalStatus()
     config = setup_config(config_path)
-    ic(config)
+    #console.log('config: ', config)
     if ignore_file_rows:
         set_ignore_file_rows = set(ignore_file_rows)
     if list_pick_columns:
@@ -127,40 +146,44 @@ def convert(
             list_actions,
             action_delimiter=action_delimiter
         )
+    writer = None
     if output_file:
-        saver = get_saver(output_file)
-    ic(config)
-    #return # debug return
+        writer = get_writer(
+            output_file,
+            progress=progress,
+        )
+    num_stacked_rows = 0
     for input_file in input_files:
-        ic(input_file)
         if not os.path.exists(input_file):
             raise FileNotFoundError(f'File not found: {input_file}')
         base_name = os.path.basename(input_file)
-        df = load(input_file, skip_header=skip_header)
-        # NOTE: NaN を None に変換しておかないと厄介
-        df = df.replace([np.nan], [None])
-        #ic(df)
-        #ic(len(df))
-        #ic(df.columns)
-        #ic(df.iloc[0])
-        #new_rows = []
-        new_flat_rows = []
-        for index, flat_row in df.iterrows():
+        loader = get_loader(
+            input_file,
+            no_header=no_header,
+            progress=progress,
+        )
+        console.log('# rows: ', len(loader))
+        for index, row in enumerate(loader):
             file_row_index = f'{input_file}:{index}'
             if file_row_index in set_ignore_file_rows:
                 continue
             short_file_row_index = f'{base_name}:{index}'
             if short_file_row_index in set_ignore_file_rows:
                 continue
-            #if flat_row.empty:
-            #    continue
-            orig_row = prepare_row(flat_row)
-            row = prepare_row(flat_row)
-            if STAGING_FIELD not in orig_row.nested:
-                set_row_staging_value(row, FILE_FIELD, input_file)
-                set_row_staging_value(row, FILE_ROW_INDEX_FIELD, file_row_index)
-                set_row_staging_value(row, ROW_INDEX_FIELD, index)
-                set_row_staging_value(row, INPUT_FIELD, orig_row.nested)
+            orig_row = row.clone()
+            if STAGING_FIELD not in row:
+                row.staging[FILE_FIELD] = input_file
+                row.staging[FILE_ROW_INDEX_FIELD] = file_row_index
+                row.staging[ROW_INDEX_FIELD] = index
+                row.staging[INPUT_FIELD] = orig_row.nested
+                if loader.extension in ['.csv', '.xlsx']:
+                    #values = []
+                    #for (key, value) in orig_row.flat.items():
+                    #    values.append(value)
+                    #if values:
+                    #    row.staging[f'{INPUT_FIELD}.__values__'] = values
+                    for key_index, (key, value) in enumerate(orig_row.flat.items()):
+                        row.staging[f'{INPUT_FIELD}.__values__.{key_index}'] = value
             if config.process.assign_array:
                 row.flat= assign_array(row.flat, config.process.assign_array)
             if config.actions:
@@ -178,32 +201,35 @@ def convert(
                 except Exception as e:
                     if verbose:
                         ic(index)
-                        ic(flat_row)
+                        #ic(flat_row)
                         ic(row.flat)
                     raise e
             if config.pick:
                 remap_columns(row, config.pick)
+            if writer is None:
+                if sys.stdout.isatty():
+                    if num_stacked_rows == 0:
+                        console.print(
+                            Panel(
+                                capture_dict(
+                                    row.nested
+                                ),
+                                title='First Row',
+                            )
+                        )
             if not output_debug:
                 pop_row_staging(row)
-            new_flat_rows.append(row.flat)
-        new_df = pd.DataFrame(new_flat_rows)
-        df_list.append(new_df)
-        # NOTE: concatの仕様が変わり、all-NAの列を含むdfを連結しようとすると警告が出るようになった
-        #if ic(new_df.dropna(axis=1, how='all').empty):
-        #    ic(new_df.dropna(axis=1, how='all'))
-        #    raise ValueError('No rows to output.')
-        #df_list.append(new_df.dropna(axis=1, how='all'))
-    all_df = pd.concat(df_list)
-    #ic(all_df)
-    ic(len(all_df))
-    #ic(all_df.columns)
-    #ic(all_df.iloc[0])
-    if output_file:
-        ic('Saving to: ', output_file)
-        saver(all_df, output_file)
-    else:
-        ic(all_df)
+            if writer:
+                writer.push_row(row)
+            else:
+                pass
+            num_stacked_rows += 1
+    console.log('Total input rows: ', num_stacked_rows)
+    if writer:
+        writer.close()
+    #else:
+    #    ic(all_df)
     if row_list_filtered_out:
         df_filtered_out = pd.DataFrame(row_list_filtered_out)
         ic('Saving filtered out to: ', output_file_filtered_out)
-        saver(df_filtered_out, output_file_filtered_out)
+        writer(df_filtered_out, output_file_filtered_out)

tabpro-0.4.4/tabpro/core/io/__init__.py

@@ -0,0 +1,18 @@
+from . extensions import io_csv
+from . extensions import io_excel
+from . extensions import io_json
+from . extensions import io_jsonl
+
+from . loader import Loader
+from . extensions.manage_writers import (
+    get_writer,
+    save,
+)
+
+get_loader = Loader
+
+__all__ = [
+    'get_loader',
+    'get_writer',
+    'save',
+]

tabpro-0.4.4/tabpro/core/io/extensions/io_csv.py

@@ -0,0 +1,104 @@
+#import pandas as pd
+
+import csv
+
+from collections import OrderedDict
+
+from rich.console import Console
+
+from tqdm.auto import tqdm
+
+from . manage_loaders import (
+    Row,
+    register_loader,
+)
+from . manage_writers import (
+    BaseWriter,
+    register_writer,
+)
+
+from ... progress import (
+    Progress,
+    track,
+)
+
+@register_loader('.csv')
+def load_csv(
+    input_file: str,
+    progress: Progress | None = None,
+    **kwargs,
+):
+    no_header = kwargs.get('no_header', False)
+    # UTF-8 with BOM
+    quiet = kwargs.get('quiet', False)
+    encoding = kwargs.get('encoding', 'utf-8-sig')
+    if progress is None:
+        console = Console()
+    else:
+        console = progress.console
+    console.log('Loading CSV data from: ', input_file)
+    def get_iter(reader):
+        return track(
+            reader,
+            description='Loading rows...',
+            disable=quiet,
+            progress=progress,
+        )
+    reader = csv.reader(open(input_file, 'r', encoding=encoding))
+    if no_header:
+        for i, row in enumerate(get_iter(reader)):
+            d = {}
+            for j, field in enumerate(row):
+                d[f'{j}'] = field
+            yield Row.from_dict(d)
+    else:
+        for i, row in enumerate(get_iter(reader)):
+            if i == 0:
+                header = row
+                continue
+            d = OrderedDict()
+            for j, field in enumerate(row):
+                d[header[j]] = field
+            #for j in range(len(row), len(header)):
+            #    d[f'__staging__.__values__.{j}'] = None
+            yield Row.from_dict(d)
+
+@register_writer('.csv')
+class CsvWriter(BaseWriter):
+    def __init__(
+        self,
+        output_file: str,
+        **kwargs,
+    ):
+        self.writer: csv.DictWriter | None = None
+        super().__init__(
+            output_file,
+            **kwargs,
+        )
+
+    def support_streaming(self):
+        return True
+
+    def _write_row(self, row: Row):
+        if self.writer is None:
+            # NOTE: 最初の行を取得するまでヘッダーを決定できない
+            self.writer = csv.DictWriter(self.fobj, fieldnames=row.flat.keys())
+            self.writer.writeheader()
+        self.writer.writerow(row.flat)
+
+    def _write_all_rows(self):
+        if self.rows is None:
+            return
+        self._open()
+        self.writer = csv.DictWriter(self.fobj)
+        header = []
+        for row in self.rows:
+            for key in row.keys():
+                if key not in header:
+                    header.append(key)
+        self.writer.writeheader(header)
+        for row in self.rows:
+            self._write_row(row)
+        self.writer = None
+        self.fobj.close()
+        self.fobj = None

tabpro-0.4.4/tabpro/core/io/extensions/io_excel.py

@@ -0,0 +1,66 @@
+from icecream import ic
+from tqdm.auto import tqdm
+import numpy as np
+import pandas as pd
+
+from rich.console import Console
+
+from logzero import logger
+
+from . manage_loaders import (
+    Row,
+    register_loader,
+)
+from . manage_writers import (
+    BaseWriter,
+    register_writer,
+)
+
+@register_loader('.xlsx')
+def load_excel(
+    input_file: str,
+    console: Console | None = None,
+    **kwargs,
+):
+    quiet = kwargs.get('quiet', False)
+    if not quiet:
+        if console is None:
+            console = Console()
+        console.log('Loading excel data from: ', input_file)
+    # NOTE: Excelで勝手に日時データなどに変換されてしまうことを防ぐため
+    df = pd.read_excel(input_file, dtype=str)
+    # NOTE: 列番号でもアクセスできるようフィールドを追加する
+    #df_with_column_number = pd.read_excel(
+    #    input_file, dtype=str, header=None, skiprows=1
+    #)
+    #new_column_names = [f'__staging__.__input__.__values__.{i}' for i in df_with_column_number.columns]
+    #df2 = df_with_column_number.rename(columns=dict(
+    #    zip(df_with_column_number.columns, new_column_names)
+    #))
+    #df = pd.concat([df, df2], axis=1)
+    #df = df.dropna(axis=0, how='all')
+    #df = df.dropna(axis=1, how='all')
+    # NOTE: NaN を None に変換しておかないと厄介
+    df = df.replace([np.nan], [None])
+    #return df
+    for i, row in df.iterrows():
+        yield Row.from_dict(row.to_dict())
+
+@register_writer('.xlsx')
+class ExcelWriter(BaseWriter):
+    def __init__(
+        self,
+        target: str,
+        **kwargs,
+    ):
+        super().__init__(target, **kwargs)
+
+    def support_streaming(self):
+        return False
+    
+    def _write_all_rows(
+        self,
+    ):
+        df = pd.DataFrame([row.flat for row in self.rows])
+        df.to_excel(self.target, index=False)
+        self.finished = True

tabpro-0.4.4/tabpro/core/io/extensions/io_json.py

@@ -0,0 +1,57 @@
+import json
+
+from rich.console import Console
+
+from . manage_loaders import (
+    Row,
+    register_loader,
+)
+from . manage_writers import (
+    BaseWriter,
+    register_writer,
+)
+
+from ... progress import Progress
+
+@register_loader('.json')
+def load_json(
+    input_file: str,
+    progress: Progress | None = None,
+    **kwargs,
+):
+    quiet = kwargs.get('quiet', False)
+    if not quiet:
+        if progress is not None:
+            console = progress.console
+        else:
+            console = Console()
+        console.log('Loading JSON data from: ', input_file)
+    with open(input_file, 'r') as f:
+        data = json.load(f)
+    if not isinstance(data, list):
+        raise ValueError(f'Invalid JSON array data: {input_file}')
+    for row in data:
+        yield Row.from_dict(row)
+
+@register_writer('.json')
+class JsonWriter(BaseWriter):
+    def __init__(
+        self,
+        output_file: str,
+        **kwargs,
+    ):
+        super().__init__(output_file, **kwargs)
+
+    def support_streaming(self):
+        return False
+    
+    def _write_all_rows(self):
+        self._open()
+        if not self.quiet:
+            console = self._get_console()
+            console.log(f'Writing {len(self.rows)} JSON rows into: ', self.target)
+        rows = [row.nested for row in self.rows]
+        self.fobj.write(json.dumps(rows, indent=2, ensure_ascii=False))
+        self.fobj.close()
+        self.fobj = None
+        self.finished = True