rda-python-dsquasar 2.0.5__tar.gz → 2.0.6__tar.gz

{rda_python_dsquasar-2.0.5 → rda_python_dsquasar-2.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 2.0.5
+Version: 2.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-2.0.5 → rda_python_dsquasar-2.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "2.0.5"
+version = "2.0.6"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-2.0.5 → rda_python_dsquasar-2.0.6}/src/rda_python_dsquasar/taccrec.py

@@ -1,3 +1,4 @@
+#!/usr/bin/env python3
 import os
 import tarfile
 import argparse
@@ -93,12 +94,12 @@ def insert_tfile_row(summary, db_params, extra, update_on_conflict=False, member
     try:
         cur.execute(sql, values)
         conn.commit()
-        # Retrieve tidx for the just-inserted tfile row
-        cur.execute(f"SELECT tidx FROM {table_name} WHERE tfile=%s", (summary['tfile'],))
+        # Retrieve tid for the just-inserted tfile row
+        cur.execute(f"SELECT tid FROM {table_name} WHERE tfile=%s", (summary['tfile'],))
         row = cur.fetchone()
-        tidx = row[0] if row else None
+        tid = row[0] if row else None
         # Update wfile tables if member_details provided
-        if member_details and tidx is not None:
+        if member_details and tid is not None:
             for m in member_details:
                 name = m['name']
                 if '/' in name:
@@ -111,8 +112,8 @@ def insert_tfile_row(summary, db_params, extra, update_on_conflict=False, member
                 exists = cur.fetchone()[0]
                 if not exists:
                     continue
-                # Update tidx if record exists
-                cur.execute(f"UPDATE {wfile_table} SET tidx=%s WHERE wfile=%s", (tidx, wfile))
+                # Update tid if record exists
+                cur.execute(f"UPDATE {wfile_table} SET tid=%s WHERE wfile=%s", (tid, wfile))
         conn.commit()
     except Exception as e:
         print(f"Database error: {e}")
@@ -144,6 +145,7 @@ def get_uid_from_logname(db_params):
 def main():
     parser = argparse.ArgumentParser(description='Insert tar file summary into tfile table.')
     parser.add_argument('--member-list', help='Path to tar member list file (from tar -tvf)')
+    parser.add_argument('--member-list-file', help='File containing list of tar member list files (one per line)')
     parser.add_argument('--db-host', default='rda-db.ucar.edu', help='Database host (default: rda-db.ucar.edu)')
     parser.add_argument('--db-port', default=5432, type=int, help='Database port (default: 5432)')
     parser.add_argument('--db-name', default='rdadb', help='Database name (default: rdadb)')
@@ -151,6 +153,55 @@ def main():
     parser.add_argument('--db-password', help='Database password (optional, use .pgpass if omitted)')
     parser.add_argument('--no-update', action='store_true', help='If tfile exists, skip all updates including wfile tables (default: False)')
     args = parser.parse_args()
+    if args.member_list_file:
+        with open(args.member_list_file, 'r') as f:
+            for line in f:
+                member_list_path = line.strip()
+                if not member_list_path or not os.path.isfile(member_list_path):
+                    print(f"Error: member list file '{member_list_path}' is invalid or does not exist.")
+                    continue
+                summary, member_details = get_tar_summary_and_details(member_list_path)
+                if summary is None:
+                    continue
+                db_params = {
+                    'host': args.db_host,
+                    'port': args.db_port,
+                    'dbname': args.db_name,
+                    'user': args.db_user
+                }
+                if args.db_password:
+                    db_params['password'] = args.db_password
+                # Check if tfile exists if --no-update is set
+                if args.no_update:
+                    try:
+                        conn = psycopg2.connect(**db_params)
+                        cur = conn.cursor()
+                        cur.execute("SELECT 1 FROM dssdb.tfile WHERE tfile=%s LIMIT 1", (summary['tfile'],))
+                        exists = cur.fetchone()
+                        cur.close()
+                        conn.close()
+                        if exists:
+                            print(f"tfile '{summary['tfile']}' already exists in dssdb.tfile. Skipping all updates.")
+                            continue
+                    except Exception as e:
+                        print(f"Database error during tfile existence check: {e}")
+                        continue
+                try:
+                    uid = get_uid_from_logname(db_params)
+                except Exception as e:
+                    print(f"Error getting uid from dssgrp: {e}")
+                    continue
+                extra = {
+                    'uid': uid,
+                    'dsid': summary['dsid'],
+                    'data_format': '',
+                    'disp_order': 0,
+                    'dsids': summary['dsids'],
+                    'note': summary['note']
+                }
+                insert_tfile_row(summary, db_params, extra, update_on_conflict=True, member_details=member_details if not args.no_update else None)
+                print(f"Inserted tar summary for {summary['tfile']} into tfile.")
+        return
     if not args.member_list or not os.path.isfile(args.member_list):
         print('Error: --member-list argument is required and must point to a valid file.')
         return

{rda_python_dsquasar-2.0.5 → rda_python_dsquasar-2.0.6}/src/rda_python_dsquasar/tacctar.py

@@ -68,25 +68,33 @@ def group_files_by_size(files, min_size, max_size):
         batches.append(current_batch)
     return batches
 
-def tar_batches(dirpath, batches, output_dir):
-    src_dir_name = Path(dirpath).name
+def tar_batches(dirpath, batches, output_dir, root_path=None, root_dirname=None):
+    # Use root_dirname for tar file naming and arcname
     for idx, batch in enumerate(batches, 1):
-        tar_name = os.path.join(output_dir, f"{src_dir_name}_part{idx}.tar")
-        logging.info(f"Creating tar: {tar_name} with {len(batch)} files.")
+        num_files = len(batch)
+        tar_name = os.path.join(output_dir, f"{root_dirname}_part{idx}_{num_files}files.tar")
+        logging.info(f"Creating tar: {tar_name} with {num_files} files.")
         with tarfile.open(tar_name, "w") as tar:
             for f in batch:
                 try:
-                    tar.add(f, arcname=os.path.relpath(f, dirpath))
+                    # arcname should be relative to root_path, and always start with root_dirname
+                    arcname = os.path.relpath(f, root_path)
+                    arcname = os.path.join(root_dirname, arcname)
+                    tar.add(f, arcname=arcname)
                 except Exception as e:
                     logging.warning(f"Failed to add {f} to tar: {e}")
 
-def process_directory_tree(root_dir, output_dir):
-    for dirpath, dirnames, filenames in os.walk(root_dir):
+def process_directory_tree(root_path, output_dir):
+    root_dirname = os.path.basename(os.path.abspath(root_path))
+    # Gather all files under root_path first
+    all_files = []
+    for dirpath, dirnames, filenames in os.walk(root_path):
         abs_files = [os.path.join(dirpath, f) for f in filenames]
-        if not abs_files:
-            continue
-        batches = group_files_by_size(abs_files, ONE_TB, THREE_TB)
-        tar_batches(dirpath, batches, output_dir)
+        all_files.extend(abs_files)
+    if not all_files:
+        return
+    batches = group_files_by_size(all_files, ONE_TB, THREE_TB)
+    tar_batches(root_path, batches, output_dir, root_path=root_path, root_dirname=root_dirname)
 
 def read_directories_from_file(input_file):
     dirs = []

{rda_python_dsquasar-2.0.5 → rda_python_dsquasar-2.0.6}/src/rda_python_dsquasar.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 2.0.5
+Version: 2.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar