rda-python-dsquasar 1.0.4__tar.gz → 1.0.6__tar.gz

{rda_python_dsquasar-1.0.4 → rda_python_dsquasar-1.0.6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 1.0.4
+Version: 1.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-1.0.4 → rda_python_dsquasar-1.0.6}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsquasar"
-version = "1.0.4"
+version = "1.0.6"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsquasar-1.0.4 → rda_python_dsquasar-1.0.6}/src/rda_python_dsquasar/dsquasar.py

@@ -83,9 +83,10 @@ SOPT = 1
 WOPT = 2
 ORDERS = {}    # cache current order number for each dsid
 BFILES = {}    # cache current order number for each dsid
-BCKSIZE = 64424509440  # 60GB, dsglobus transfer size
-TARSIZE = 3221225472   # 3GB, tarfile size limit
-MINSIZE = PgLOG.PGLOG['ONEGBS']   # minimal tarfile size
+BCKSIZE = 90*PgLOG.PGLOG['ONEGBS']  # 90GB, dsglobus transfer size
+TARSIZE = 5*PgLOG.PGLOG['ONEGBS']   # 5GB, tarfile size limit
+MINSIZE = PgLOG.PGLOG['TWOGBS']     # 2GB, minimal tarfile size
+ONESIZE = 20*PgLOG.PGLOG['ONEGBS']  # 20GB, minimal file size to tar a single file 
 TFCOUNT = 100          # if file count is greater, use MINSIZE for tar file
 SUBLMTS = 2000        # file count limit for a sub-group
 
@@ -899,7 +900,9 @@ def process_backup_files(qinfo, dsid, fcnt, recs, filetype):
       if not fd: fd = open_input_file(qinfo, dsid, filetype)
       fd.write(instr)
       qinfo['instr'] += instr
-      if qinfo['fcnt'] < TFCOUNT:
+      if qinfo['fcnt'] == 1:
+         if qinfo['size'] < ONESIZE: continue
+      elif qinfo['fcnt'] < TFCOUNT:
          if qinfo['size'] < TARSIZE: continue
       else:
          if qinfo['size'] < MINSIZE: continue
@@ -1173,7 +1176,7 @@ def gather_dataset_files(dsfiles, dsids, unlock = True):
          dcnd = "dsid = '{}'".format(dsid)
          pgrec = PgDBI.pgget("dataset", "dsid, backflag, pid", dcnd, PgLOG.LGWNEX)
          if pgrec:
-            if unlock and pgrec['pid'] and PgLock.lock_dataset(dsid, 0, PgLOG.LGEREX) < 1: continue
+            if unlock and pgrec['pid'] and PgLock.lock_dataset(dsid, 0, LOGACT) < 1: continue
             fcnt += get_dataset_files(dsid, dsfiles, pgrec['backflag'], PgLOG.LOGWRN)
    else:
       dcnd = "ORDER BY dsid"
@@ -1181,7 +1184,7 @@ def gather_dataset_files(dsfiles, dsids, unlock = True):
       dcnt = len(pgrecs['dsid']) if pgrecs else 0
       for i in range(dcnt):
          dsid = pgrecs['dsid'][i]
-         if unlock and pgrecs['pid'][i] and PgLock.lock_dataset(dsid, 0, PgLOG.LGEREX) < 1: continue
+         if unlock and pgrecs['pid'][i] and PgLock.lock_dataset(dsid, 0, LOGACT) < 1: continue
          fcnt += get_dataset_files(dsid, dsfiles, pgrecs['backflag'][i])
 
    if dsfiles:

{rda_python_dsquasar-1.0.4 → rda_python_dsquasar-1.0.6}/src/rda_python_dsquasar.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsquasar
-Version: 1.0.4
+Version: 1.0.6
 Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar

{rda_python_dsquasar-1.0.4 → rda_python_dsquasar-1.0.6}/tests/test_dsquasar.py

@@ -1,4 +1,4 @@
-# test_hello_world.py
+# test
 
 import pytest