python-table-processor 0.3.8__py3-none-any.whl → 0.3.9__py3-none-any.whl

{python_table_processor-0.3.8.dist-info → python_table_processor-0.3.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: python-table-processor
-Version: 0.3.8
+Version: 0.3.9
 Summary: A table data processor
 Home-page: https://github.com/akivajp/python-table-processor
 License: MIT

{python_table_processor-0.3.8.dist-info → python_table_processor-0.3.9.dist-info}/RECORD

@@ -1,7 +1,7 @@
-table_processor/__init__.py,sha256=2aCcYBh8v0QBhm77EG8Rb7HSYaY18N0gWeE08uXri3o,52
+table_processor/__init__.py,sha256=sA4b5QjPy1l9_d8B5UqNxW9ElcTD5ccsduZThd_EcI0,52
 table_processor/cli.py,sha256=uluf_VsBzq3Ldxh3ePGseSrv8--lI7huu-6RjDaHPPg,2170
 table_processor/commands/convert_tables.py,sha256=w4IZftSlrbt-BMPjldULT_616LSiNE3h7GmrFF7K5OM,2276
-table_processor/commands/merge_tables.py,sha256=jryGU3gUx2TlGrI6FozGS_aJQvJQFco2LKwun6Uo9KY,1719
+table_processor/commands/merge_tables.py,sha256=GHz3uBj_yeQzBr1_2b2ua8EUr0_B3_M-33CSdTD_wUc,1974
 table_processor/core/actions.py,sha256=BNXf9iptGl9xV2itNK4bVx1jjVHCiRgTIFlZyULe6Dc,23695
 table_processor/core/config.py,sha256=Z1s2r3y4oQXdbDCwIY_hYzoq6VRv7Sb3oUlfBCu13Lc,11646
 table_processor/core/constants.py,sha256=w-1TxtpXywb6Qh8M4YWuzcCHUvCjy5zJYxaXC3RQS6A,158
@@ -14,10 +14,10 @@ table_processor/core/functions/search_column_value.py,sha256=GscwLaMTORHFDhdCdjz
 table_processor/core/functions/set_flat_field_value.py,sha256=7-BgD2yEUGyBaFaIJ18bYv82Qdy0CN8NiC5ZtsAdv44,531
 table_processor/core/functions/set_nested_field_value.py,sha256=351AoyL8-dT9I4eBKYlxUME03ACiMjDukebzmhOBaks,1308
 table_processor/core/functions/set_row_value.py,sha256=wlF_nVGZY74XCzd6pBJJyti62m6y0s1kc5Q6uq9jn8o,641
-table_processor/core/merge.py,sha256=5U6fvofaMq6yvllB24M6Pf3uG7Hz17aLKM47NmWHJBc,5027
+table_processor/core/merge.py,sha256=mP-DR7Hb6RJcqXEyEgk33AWUYlOgdcvLVYY-IsMKr9A,5361
 table_processor/core/types.py,sha256=xNvy1vL1ei87kt-MgSfRJSllYrNKRo23pHtJA15DyLQ,2895
-python_table_processor-0.3.8.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
-python_table_processor-0.3.8.dist-info/METADATA,sha256=JbejFTK1nVvFUQuI4XfO9ei02fdigXsBujLnQNAKges,1098
-python_table_processor-0.3.8.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
-python_table_processor-0.3.8.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
-python_table_processor-0.3.8.dist-info/RECORD,,
+python_table_processor-0.3.9.dist-info/LICENSE,sha256=1_Gn0I1neLPxDLfLiHEyxjDg-pbAra1p2iHEqutGS_c,1068
+python_table_processor-0.3.9.dist-info/METADATA,sha256=RsoTUywEZBrBs1_XiEJO05LVNHHJaEnxVyzekwfLC_0,1098
+python_table_processor-0.3.9.dist-info/WHEEL,sha256=sP946D7jFCHeNz5Iq4fL4Lu-PrWrFsgfLXbbkciIZwg,88
+python_table_processor-0.3.9.dist-info/entry_points.txt,sha256=lq97F6m51yZ93qmVIqkUHJfbILZuYLxrMWXSqrP4JpM,118
+python_table_processor-0.3.9.dist-info/RECORD,,

table_processor/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.3.8"
-__version_tuple__ = (0, 3, 8)
+__version__ = "0.3.9"
+__version_tuple__ = (0, 3, 9)

table_processor/commands/merge_tables.py

@@ -17,6 +17,7 @@ def run(
         ignore_not_found=args.ignore_not_found,
         output_base_data_file=args.output_base_data_file,
         output_modified_data_file=args.output_modified_data_file,
+        output_remaining_data_file=args.output_remaining_data_file,
     )
 
 def setup_parser(
@@ -60,4 +61,9 @@ def setup_parser(
         required=False,
         help='Path to output modified data file',
     )
+    parser.add_argument(
+        '--output-remaining-data-file', '--output-remaining', '--output-remain',
+        required=False,
+        help='Path to output remaining data file',
+    )
     parser.set_defaults(handler=run)

table_processor/core/merge.py

@@ -39,6 +39,19 @@ from . convert import (
     save,
 )
 
+def get_primary_key(
+    row: Mapping,
+    keys: list[str],
+):
+    list_keys = []
+    for key in keys:
+        value, found = search_column_value(row, key)
+        if not found:
+            raise KeyError(f'Column not found: {key}, existing columns: {row.keys()}')
+        list_keys.append(value)
+    primary_key = tuple(list_keys)
+    return primary_key
+
 def merge(
     previous_files: list[str],
     modification_files: list[str],
@@ -47,13 +60,16 @@ def merge(
     ignore_not_found: bool = False,
     output_base_data_file: str | None = None,
     output_modified_data_file: str | None = None,
+    output_remaining_data_file: str | None = None,
 ):
     ic.enable()
     ic()
     ic(previous_files)
     ic(modification_files)
     ic(keys)
-    dict_key_to_row = {}
+    #dict_key_to_row = {}
+    dict_key_to_row = OrderedDict()
+    set_modified_keys = set()
     all_base_rows = []
     all_modified_rows = []
     list_ignored_keys = []
@@ -74,13 +90,7 @@ def merge(
             total=len(df),
         ):
             row = prepare_row(flat_row)
-            list_keys = []
-            for key in keys:
-                value, found = search_column_value(row.nested, key)
-                if not found:
-                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
-                list_keys.append(value)
-            primary_key = tuple(list_keys)
+            primary_key = get_primary_key(row.flat, keys)
             #ic(key)
             if not allow_duplicate_keys:
                 if primary_key in dict_key_to_row:
@@ -104,13 +114,7 @@ def merge(
             total=len(df),
         ):
             row = prepare_row(flat_row)
-            list_keys = []
-            for key in keys:
-                value, found = search_column_value(row.nested, key)
-                if not found:
-                    raise KeyError(f'Column not found: {key}, existing columns: {row.flat.keys()}')
-                list_keys.append(value)
-            primary_key = tuple(list_keys)
+            primary_key = get_primary_key(row.flat, keys)
             #ic(key)
             #if key not in dict_key_to_row:
             #    dict_key_to_row[key] = row
@@ -137,6 +141,7 @@ def merge(
             #ic(previous_row)
             #ic(previous_row.flat)
             #raise
+            set_modified_keys.add(primary_key)
             num_modified += 1
     ic(num_modified)
     if ignore_not_found:
@@ -150,3 +155,11 @@ def merge(
         all_df = pd.DataFrame([row.flat for row in all_modified_rows])
         ic('Saving to: ', output_modified_data_file)
         save(all_df, output_modified_data_file)
+    if output_remaining_data_file:
+        remaining_rows = []
+        for key, row in dict_key_to_row.items():
+            if key not in set_modified_keys:
+                remaining_rows.append(row)
+        all_df = pd.DataFrame([row.flat for row in remaining_rows])
+        ic('Saving to: ', output_remaining_data_file)
+        save(all_df, output_remaining_data_file)