esrf-data-compressor 0.1.0__tar.gz → 0.1.2__tar.gz

{esrf_data_compressor-0.1.0/src/esrf_data_compressor.egg-info → esrf_data_compressor-0.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esrf-data-compressor
-Version: 0.1.0
+Version: 0.1.2
 Summary: A library to compress ESRF data and reduce their footprint
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: MIT License
@@ -90,10 +90,10 @@ Dynamic: license-file
 
 * **Four simple CLI subcommands**
 
-  * `list`  Show all raw HDF5 files to be processed
-  * `compress` Generate compressed siblings
-  * `check`  Produce a per-dataset SSIM report between raw & compressed
-  * `overwrite` Atomically replace each raw frame file (irreversible)
+  * `compress-hdf5 list`  Show all raw HDF5 files to be processed
+  * `compress-hdf5 compress` Generate compressed siblings
+  * `compress-hdf5 check`  Produce a per-dataset SSIM report between raw & compressed
+  * `compress-hdf5 overwrite` Atomically replace each raw frame file (irreversible)
 
 ---
 
@@ -177,7 +177,7 @@ All noteworthy changes are recorded in [CHANGELOG.md](CHANGELOG.md). Version 0.1
 
 * Initial implementation of Blosc2 + Grok (JPEG2000) compression for 3D HDF5 datasets.
 * SSIM-based integrity check (first & last slice).
-* Four-command CLI (`list`, `compress`, `check`, `overwrite`).
+* Four-command CLI (`compress-hdf5 list`, `compress-hdf5 compress`, `compress-hdf5 check`, `compress-hdf5 overwrite`).
 * Parallelism with worker×thread auto-factoring.
 
 For more details, see the full history in [CHANGELOG.md](CHANGELOG.md).

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/README.md

@@ -29,10 +29,10 @@
 
 * **Four simple CLI subcommands**
 
-  * `list`  Show all raw HDF5 files to be processed
-  * `compress` Generate compressed siblings
-  * `check`  Produce a per-dataset SSIM report between raw & compressed
-  * `overwrite` Atomically replace each raw frame file (irreversible)
+  * `compress-hdf5 list`  Show all raw HDF5 files to be processed
+  * `compress-hdf5 compress` Generate compressed siblings
+  * `compress-hdf5 check`  Produce a per-dataset SSIM report between raw & compressed
+  * `compress-hdf5 overwrite` Atomically replace each raw frame file (irreversible)
 
 ---
 
@@ -116,7 +116,7 @@ All noteworthy changes are recorded in [CHANGELOG.md](CHANGELOG.md). Version 0.1
 
 * Initial implementation of Blosc2 + Grok (JPEG2000) compression for 3D HDF5 datasets.
 * SSIM-based integrity check (first & last slice).
-* Four-command CLI (`list`, `compress`, `check`, `overwrite`).
+* Four-command CLI (`compress-hdf5 list`, `compress-hdf5 compress`, `compress-hdf5 check`, `compress-hdf5 overwrite`).
 * Parallelism with worker×thread auto-factoring.
 
 For more details, see the full history in [CHANGELOG.md](CHANGELOG.md).

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "esrf-data-compressor"
-version = "0.1.0"
+version = "0.1.2"
 authors = [{ name = "ESRF", email = "dau-pydev@esrf.fr" }]
 description = "A library to compress ESRF data and reduce their footprint"
 readme = { file = "README.md", content-type = "text/markdown" }

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/src/esrf_data_compressor/cli.py

@@ -46,7 +46,7 @@ def do_compress(args):
         exit_with_error(f"Failed to read report '{report}': {e}")
 
     if not files:
-        print("Nothing to compress (TO COMPRESS list is empty).")
+        print("Nothing to compress (TO COMPRESS list is empty).")
         return
 
     print(
@@ -65,10 +65,9 @@ def do_check(args):
         exit_with_error(f"Failed to read report '{report}': {e}")
 
     if not files:
-        print("Nothing to check (TO COMPRESS list is empty).")
+        print("Nothing to check (TO COMPRESS list is empty).")
         return
 
-    # We reuse run_ssim_check in its 3‑arg form (raw_files, method, report_path)
     report_fname = f"{os.path.splitext(report)[0]}_{args.method}_ssim_report.txt"
     report_path = os.path.abspath(report_fname)
 
@@ -81,9 +80,6 @@ def do_check(args):
 
 
 def do_overwrite(args):
-    """
-    Overwrite TO COMPRESS files with their original sources.
-    """
     report = args.input or "file_list.txt"
     try:
         files = parse_report(report)
@@ -91,13 +87,26 @@ def do_overwrite(args):
         exit_with_error(f"Failed to read report '{report}': {e}")
 
     if not files:
-        print("Nothing to overwrite (TO COMPRESS list is empty).")
+        print("Nothing to process (TO COMPRESS list is empty).")
         return
 
-    print(f"Overwriting {len(files)} file(s) from '{report}' …")
     mgr = CompressorManager()
+
+    if args.final:
+        print(f"Finalizing overwrite for {len(files)} file(s) from '{report}' …")
+        mgr.remove_backups(files)
+        print("Finalize step complete.\n")
+        return
+
+    if args.undo:
+        print(f"Undoing overwrite for {len(files)} file(s) from '{report}' …")
+        mgr.restore_backups(files)
+        print("Undo step complete.\n")
+        return
+
+    print(f"Overwriting {len(files)} file(s) from '{report}' …")
     mgr.overwrite_files(files)
-    print("Overwrite complete.\n")
+    print("Overwrite complete (backups kept).\n")
 
 
 def main():
@@ -106,7 +115,6 @@ def main():
     )
     sub = parser.add_subparsers(dest="command", required=True)
 
-    # list
     p = sub.add_parser("list", help="Report VDS sources → TO COMPRESS vs REMAINING")
     p.add_argument("experiment", help="Experiment ID")
     p.add_argument("beamline", nargs="?", help="Optional beamline")
@@ -115,13 +123,12 @@ def main():
     p.add_argument(
         "--filter",
         metavar="KEY:VAL[,KEY2:VAL2...]",
-        help="Dataset‑level attribute substring filters",
+        help="Dataset-level attribute substring filters",
     )
     p.add_argument("--output", help="Report file (default = file_list.txt)")
     p.set_defaults(func=do_list)
 
-    # compress
-    p = sub.add_parser("compress", help="Compress only the TO COMPRESS files")
+    p = sub.add_parser("compress", help="Compress only the TO COMPRESS files")
     p.add_argument(
         "--input",
         "-i",
@@ -137,8 +144,7 @@ def main():
     )
     p.set_defaults(func=do_compress)
 
-    # check
-    p = sub.add_parser("check", help="Generate SSIM report for TO COMPRESS files")
+    p = sub.add_parser("check", help="Generate SSIM report for TO COMPRESS files")
     p.add_argument(
         "--input", "-i", help="Report file to read (default = file_list.txt)"
     )
@@ -147,11 +153,24 @@ def main():
     )
     p.set_defaults(func=do_check)
 
-    # overwrite
-    p = sub.add_parser("overwrite", help="Overwrite only TO COMPRESS files")
+    p = sub.add_parser(
+        "overwrite",
+        help="Swap in compressed files and keep backups; with --final or --undo, perform cleanup/restore only.",
+    )
     p.add_argument(
         "--input", "-i", help="Report file to read (default = file_list.txt)"
     )
+    group = p.add_mutually_exclusive_group()
+    group.add_argument(
+        "--final",
+        action="store_true",
+        help="Cleanup only: delete existing *.h5.bak backups after confirmation (no overwrite).",
+    )
+    group.add_argument(
+        "--undo",
+        action="store_true",
+        help="Restore only: move <file>.h5.bak back to <file>.h5 and preserve the current file as <file>_<method>.h5 when needed.",
+    )
     p.set_defaults(func=do_overwrite)
 
     args = parser.parse_args()

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/src/esrf_data_compressor/compressors/base.py

@@ -34,24 +34,14 @@ class CompressorManager:
         self, workers: int | None = None, cratio: int = 10, method: str = "jp2k"
     ):
         total_cores = os.cpu_count() or 1
-
-        # Determine default threads per worker (4, or fewer if total_cores < 4)
-        if total_cores >= 4:
-            default_nthreads = 4
-        else:
-            default_nthreads = 1
-
-        # Default worker count
+        default_nthreads = 4 if total_cores >= 4 else 1
         default_workers = max(1, total_cores // default_nthreads)
 
         if workers is None:
-            # Use default workers and default_nthreads
             w = default_workers
             nthreads = default_nthreads
         else:
-            # Cap workers to total_cores
             w = min(workers, total_cores)
-            # Recompute threads per worker so that (w * nthreads) ≤ total_cores, up to 4
             possible = total_cores // w
             nthreads = min(possible, 4) if possible >= 1 else 1
 
@@ -60,7 +50,6 @@ class CompressorManager:
         self.cratio = cratio
         self.method = method
 
-        # Instantiate compressor based on method
         if self.method == "jp2k":
             self.compressor = JP2KCompressorWrapper(
                 cratio=cratio, nthreads=self.nthreads
@@ -122,8 +111,7 @@ class CompressorManager:
                 except Exception as e:
                     print(f"Failed to compress '{pth}': {e}")
 
-        t1 = time.time()
-        elapsed = t1 - t0
+        elapsed = time.time() - t0
         total_mb = total_bytes / (1024 * 1024)
         rate_mb_s = total_mb / elapsed if elapsed > 0 else float("inf")
         print(f"\nTotal elapsed time: {elapsed:.3f}s")
@@ -138,7 +126,6 @@ class CompressorManager:
 
         After processing all files, removes the backup .h5.bak files.
         """
-        backups = []
         for ipath in file_list:
             if not ipath.lower().endswith(".h5"):
                 continue
@@ -151,17 +138,76 @@ class CompressorManager:
                 try:
                     os.replace(ipath, backup)
                     os.replace(compressed_path, ipath)
-                    backups.append(backup)
                     print(f"Overwritten '{ipath}' (backup at '{backup}').")
                 except Exception as e:
                     print(f"ERROR overwriting '{ipath}': {e}")
             else:
                 print(f"SKIP (no compressed file): {ipath}")
 
-        # Remove all backup files
-        for backup in backups:
+    def remove_backups(self, file_list: list[str]) -> None:
+        candidates = {p + ".bak" for p in file_list if p.lower().endswith(".h5")}
+        backups = [b for b in candidates if os.path.exists(b)]
+        if not backups:
+            print("No backup files to remove.")
+            return
+
+        total_bytes = 0
+        for b in backups:
+            try:
+                total_bytes += os.path.getsize(b)
+            except OSError:
+                pass
+        total_mb = total_bytes / (1024 * 1024)
+
+        print(
+            f"About to remove {len(backups)} backup file(s), ~{total_mb:.2f} MB total."
+        )
+        ans = input("Proceed? [y/N]: ").strip().lower()
+        if ans not in ("y", "yes"):
+            print("Backups kept.")
+            return
+
+        removed = 0
+        for b in backups:
             try:
-                os.remove(backup)
-                print(f"Deleted backup '{backup}'.")
+                os.remove(b)
+                removed += 1
             except Exception as e:
-                print(f"ERROR deleting backup '{backup}': {e}")
+                print(f"ERROR deleting backup '{b}': {e}")
+
+        print(f"Deleted {removed} backup file(s).")
+
+    def restore_backups(self, file_list: list[str]) -> None:
+        restored = 0
+        preserved = 0
+        for ipath in file_list:
+            if not ipath.lower().endswith(".h5"):
+                continue
+
+            base, _ = os.path.splitext(ipath)
+            backup = ipath + ".bak"
+            method_path = f"{base}_{self.method}.h5"
+
+            if not os.path.exists(backup):
+                print(f"SKIP (no backup): {ipath}")
+                continue
+
+            if os.path.exists(ipath) and not os.path.exists(method_path):
+                try:
+                    os.replace(ipath, method_path)
+                    preserved += 1
+                    print(f"Preserved current file to '{method_path}'.")
+                except Exception as e:
+                    print(f"ERROR preserving current '{ipath}' to '{method_path}': {e}")
+                    continue
+
+            try:
+                os.replace(backup, ipath)
+                restored += 1
+                print(f"Restored '{ipath}' from backup.")
+            except Exception as e:
+                print(f"ERROR restoring '{ipath}' from '{backup}': {e}")
+
+        print(
+            f"Restore complete. Restored: {restored}, preserved compressed copies: {preserved}."
+        )

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/src/esrf_data_compressor/finder/finder.py

@@ -3,7 +3,19 @@ import sys
 import re
 import h5py
 import h5py.h5d as h5d
-from typing import List, Tuple, Optional
+from typing import List, Tuple, Optional, Set, Dict
+from concurrent.futures import ProcessPoolExecutor, as_completed
+
+try:
+    from tqdm.auto import tqdm
+except Exception:
+
+    def tqdm(iterable=None, **kwargs):
+        return iterable if iterable is not None else range(0)
+
+
+JP2K_FILTER_ID = 32026
+DATASET_CHECK_PATH = "/entry_0000/measurement/data"
 
 
 def discover_datasets(path_components: List[str], base_root: str) -> List[str]:
@@ -43,8 +55,27 @@ def discover_datasets(path_components: List[str], base_root: str) -> List[str]:
     return sorted(datasets)
 
 
+def _file_has_jp2k_filter(file_path: str) -> bool:
+    try:
+        with h5py.File(file_path, "r", locking=False) as src:
+            obj = src.get(DATASET_CHECK_PATH)
+            if not isinstance(obj, h5py.Dataset):
+                return False
+            plist = obj.id.get_create_plist()
+            for j in range(plist.get_nfilters()):
+                if plist.get_filter(j)[0] == JP2K_FILTER_ID:
+                    return True
+            return False
+    except Exception:
+        return False
+
+
 def find_vds_files(
-    path_components: List[str], base_root: str, filter_expr: Optional[str]
+    path_components: List[str],
+    base_root: str,
+    filter_expr: Optional[str],
+    *,
+    max_workers: Optional[int] = None,
 ) -> Tuple[List[Tuple[str, str]], List[Tuple[str, str]]]:
     """
     Discover each dataset HDF5, then for each top-level group (e.g. "1.1"):
@@ -59,7 +90,6 @@ def find_vds_files(
 
     Returns two lists of (vds_source_path, reason).
     """
-    # parse filter tokens
     filters: List[Tuple[List[str], str]] = []
     if filter_expr:
         for tok in filter_expr.split(","):
@@ -77,13 +107,16 @@ def find_vds_files(
 
     datasets = discover_datasets(path_components, base_root)
 
-    for cont_path in datasets:
-        with h5py.File(cont_path, "r") as f:
+    unique_files: Set[str] = set()
+    occurrences: List[Tuple[str, bool, str]] = []
+
+    for cont_path in tqdm(datasets, desc="Scan datasets", unit="file"):
+        cont_dir = os.path.dirname(cont_path)
+        with h5py.File(cont_path, "r", locking=False) as f:
             for grp_name, grp in f.items():
                 if not isinstance(grp, h5py.Group):
                     continue
 
-                # determine if group matches filter criteria
                 group_matched = False
                 reason = ""
                 for parts, desired in filters:
@@ -102,55 +135,52 @@ def find_vds_files(
                 if not filters:
                     reason = f"{grp_name}/<no filter>"
 
-                # harvest VDS sources under this group, detecting existing compression
                 def visitor(name, obj):
                     if not isinstance(obj, h5py.Dataset):
                         return
                     plist = obj.id.get_create_plist()
                     if plist.get_layout() != h5d.VIRTUAL:
                         return
-
                     for i in range(plist.get_virtual_count()):
                         fn = plist.get_virtual_filename(i)
+                        if isinstance(fn, bytes):
+                            fn = fn.decode("utf-8", "ignore")
                         if not os.path.isabs(fn):
-                            fn = os.path.abspath(
-                                os.path.join(os.path.dirname(cont_path), fn)
-                            )
-
-                        # inspect file for Blosc2/Grok (JP2K) compression filter
-                        already_compressed = False
-                        try:
-                            with h5py.File(fn, "r") as src:
-                                comp_flag = [False]
-
-                                def _check(name2, obj2):
-                                    if isinstance(obj2, h5py.Dataset):
-                                        plist2 = obj2.id.get_create_plist()
-                                        for j in range(plist2.get_nfilters()):
-                                            fid = plist2.get_filter(j)[0]
-                                            if fid == 32026:
-                                                comp_flag[0] = True
-                                                return
-
-                                src.visititems(_check)
-                                already_compressed = comp_flag[0]
-                        except Exception:
-                            already_compressed = False
-
-                        if already_compressed:
-                            rem_reason = f"{grp_name}/<already compressed>"
-                            remaining.append((fn, rem_reason))
-                        else:
-                            if group_matched:
-                                to_compress.append((fn, reason))
-                            else:
-                                remaining.append((fn, reason))
+                            fn = os.path.abspath(os.path.join(cont_dir, fn))
+                        unique_files.add(fn)
+                        occurrences.append((fn, group_matched, reason))
 
                 grp.visititems(visitor)
 
-    if not to_compress and not remaining:
+    if not occurrences:
         sys.exit(f"ERROR: No VDS sources found under {base_root}/{path_components}")
 
+    fps = sorted(unique_files)
+    if max_workers is None:
+        cpu = os.cpu_count() or 8
+        max_workers = min(16, max(4, cpu))
+
+    compressed_map: Dict[str, bool] = {}
+    with ProcessPoolExecutor(max_workers=max_workers) as ex:
+        futs = {ex.submit(_file_has_jp2k_filter, fp): fp for fp in fps}
+        for fut in tqdm(
+            as_completed(futs), total=len(fps), desc="Check compression", unit="file"
+        ):
+            fp = futs[fut]
+            try:
+                compressed_map[fp] = bool(fut.result())
+            except Exception:
+                compressed_map[fp] = False
+
+    for fp, matched, reason in occurrences:
+        if compressed_map.get(fp, False):
+            remaining.append((fp, "<already compressed>"))
+        else:
+            if matched:
+                to_compress.append((fp, reason))
+            else:
+                remaining.append((fp, reason))
+
     return to_compress, remaining
 
 

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2}/src/esrf_data_compressor/tests/test_cli.py

@@ -119,12 +119,14 @@ def test_commands_with_non_empty_list(
         for f in files:
             comp = tmp_path / f.replace(".h5", "_jp2k.h5")
             assert comp.exists()
-    # For overwrite, verify original replaced and backup removed
+    # For overwrite, verify original replaced and backup KEPT
     if cmd == "overwrite":
         # f1 was overwritten, f2 was skipped
         assert (tmp_path / "f1.h5").exists()
-        # no backup remains
-        assert not (tmp_path / "f1.h5.bak").exists()
+        # backup remains by default
+        assert (tmp_path / "f1.h5.bak").exists()
+        # f2 had no compressed sibling → no backup
+        assert not (tmp_path / "f2.h5.bak").exists()
 
 
 def test_list_success_and_output_file(argv_runner, monkeypatch, capsys, tmp_path):
@@ -150,7 +152,7 @@ def test_list_success_and_output_file(argv_runner, monkeypatch, capsys, tmp_path
     [
         ("compress", "Nothing to compress"),
         ("check", "Nothing to check"),
-        ("overwrite", "Nothing to overwrite"),
+        ("overwrite", "Nothing to process"),
     ],
 )
 def test_empty_reports(argv_runner, monkeypatch, capsys, cmd, empty_msg, tmp_path):
@@ -174,3 +176,37 @@ def test_check_success_writes_report(argv_runner, monkeypatch, capsys, tmp_path)
     argv_runner(["check", "-i", str(report), "--method", "jp2k"])
     out = capsys.readouterr().out
     assert "SSIM report written to" in out
+
+
+def test_overwrite_final_deletes_backups(argv_runner, monkeypatch, capsys, tmp_path):
+    # Prepare a file and its backup
+    (tmp_path / "f1.h5").write_text("current")
+    (tmp_path / "f1.h5.bak").write_text("backup")
+    # parse_report returns the original .h5 path(s)
+    monkeypatch.setattr(cli, "parse_report", lambda rpt: [str(tmp_path / "f1.h5")])
+    # auto-confirm deletion
+    monkeypatch.setattr("builtins.input", lambda *a, **k: "y")
+
+    argv_runner(["overwrite", "-i", "report.txt", "--final"])
+    out = capsys.readouterr().out
+    assert "About to remove" in out
+    assert not (tmp_path / "f1.h5.bak").exists()
+
+
+def test_overwrite_undo_restores_and_preserves(
+    argv_runner, monkeypatch, capsys, tmp_path
+):
+    # Start with current file and a backup; no <method> file yet
+    (tmp_path / "f1.h5").write_text("CUR")
+    (tmp_path / "f1.h5.bak").write_text("BAK")
+    monkeypatch.setattr(cli, "parse_report", lambda rpt: [str(tmp_path / "f1.h5")])
+
+    argv_runner(["overwrite", "-i", "report.txt", "--undo"])
+    out = capsys.readouterr().out
+    assert "Undoing overwrite" in out
+    # Backup should have been restored to f1.h5
+    assert (tmp_path / "f1.h5").read_text() == "BAK"
+    # Previous current should have been preserved as f1_jp2k.h5
+    assert (tmp_path / "f1_jp2k.h5").read_text() == "CUR"
+    # .bak should be gone after restore (moved)
+    assert not (tmp_path / "f1.h5.bak").exists()

{esrf_data_compressor-0.1.0 → esrf_data_compressor-0.1.2/src/esrf_data_compressor.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: esrf-data-compressor
-Version: 0.1.0
+Version: 0.1.2
 Summary: A library to compress ESRF data and reduce their footprint
 Author-email: ESRF <dau-pydev@esrf.fr>
 License: MIT License
@@ -90,10 +90,10 @@ Dynamic: license-file
 
 * **Four simple CLI subcommands**
 
-  * `list`  Show all raw HDF5 files to be processed
-  * `compress` Generate compressed siblings
-  * `check`  Produce a per-dataset SSIM report between raw & compressed
-  * `overwrite` Atomically replace each raw frame file (irreversible)
+  * `compress-hdf5 list`  Show all raw HDF5 files to be processed
+  * `compress-hdf5 compress` Generate compressed siblings
+  * `compress-hdf5 check`  Produce a per-dataset SSIM report between raw & compressed
+  * `compress-hdf5 overwrite` Atomically replace each raw frame file (irreversible)
 
 ---
 
@@ -177,7 +177,7 @@ All noteworthy changes are recorded in [CHANGELOG.md](CHANGELOG.md). Version 0.1
 
 * Initial implementation of Blosc2 + Grok (JPEG2000) compression for 3D HDF5 datasets.
 * SSIM-based integrity check (first & last slice).
-* Four-command CLI (`list`, `compress`, `check`, `overwrite`).
+* Four-command CLI (`compress-hdf5 list`, `compress-hdf5 compress`, `compress-hdf5 check`, `compress-hdf5 overwrite`).
 * Parallelism with worker×thread auto-factoring.
 
 For more details, see the full history in [CHANGELOG.md](CHANGELOG.md).