rda-python-dsarch 3.0.13__tar.gz → 3.0.14__tar.gz

{rda_python_dsarch-3.0.13/src/rda_python_dsarch.egg-info → rda_python_dsarch-3.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 3.0.13
+Version: 3.0.14
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch

{rda_python_dsarch-3.0.13 → rda_python_dsarch-3.0.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsarch"
-version = "3.0.13"
+version = "3.0.14"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsarch-3.0.13 → rda_python_dsarch-3.0.14}/src/rda_python_dsarch/dsarch.usg

@@ -622,13 +622,13 @@ GroupIndex<:>GroupName<:>ParentIndex<:>Title<:>GroupPattern<:>BackupFlag<:>Saved
 2<:>MONTHLY<:>0<:>MONTHLY OAFlux Products<:><:>P<:><:>monthly<:>
 2 group records retrieved
 d260001-G1: DS-11094192261 DC-327 WC-327 SS-8909173908 SC-51
-d260001-G1 Web Files: Total 327(11094192261), Missing 0(0), Quasar Backup&DR 327(11094192261)
-d260001-G1 Saved Files: Total 51(8909173908), Missing 0(0), Quasar Backup&DR 51(8909173908)
+d260001-G1 Web Files: Total 327(11.09GB), Quasar Backup&DR 327(11.09GB)
+d260001-G1 Saved Files: Total 51(8.91GB), Quasar Backup&DR 51(8.91GB)
 d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
-d260001-G2 Web Files: Total 522(552780725), Missing 2(2116424), Missing but on Quasar 1(1058212), Missing on Glade 2(2116424), Quasar Backup&DR 521(551722513), Not Backed Up 1(1058212)
+d260001-G2 Web Files: Total 522(552.78MB), Missing on Glade 2(2.12MB) [Quasar Backup&DR 1(1.06MB), Not Backed Up 1(1.06MB)], Quasar Backup&DR 521(551.72MB), Not Backed Up 1(1.06MB)
 d260001-G2-W-D: monthly/2025/oaflux_202511.nc: Missing on Glade; on Quasar Backup&DR (Backup ID 3456789)
 d260001-G2-W-D: monthly/2025/oaflux_202512.nc: Missing on Glade; Not Backed Up
-d260001-G2 Saved Files: Total 51(465789806), Missing 0(0), Quasar Backup&DR 51(465789806)
+d260001-G2 Saved Files: Total 51(465.79MB), Quasar Backup&DR 51(465.79MB)
 
 
 3.3.3 Delete Group Information
@@ -1941,14 +1941,16 @@ behavior.
   Files with Location Flag C (CGD) or R (remote) are not stored by
   GDEX and are not checked. One summary line per file category shows
   the file counts with -WN and the total file sizes with -WS (both as
-  Count(Size) when both options are given): the Total, the total
-  Missing (each file counted once; a Saved file found in COLD_STORAGE
-  is not missing), Missing but on Quasar (missing files that can still
-  be restored from a finished Quasar backup with action -RQ, so Missing
-  minus it is what is lost), then the result of each check. It is
-  followed by one line for every file missing or not backed up yet;
-  the line of a missing file also tells its Quasar backup, or that it
-  is not backed up.
+  Count(Size) when both options are given, sizes in KB/MB/GB/TB): the
+  Total, then the result of each check. Each Missing on Glade/Boreas
+  count is followed by the Quasar status of its files in [], e.g.
+  [Quasar Backup&DR 1(1.06MB), Not Backed Up 1(1.06MB)], so it tells
+  how many of the missing files can be restored with action -RQ and
+  how many are lost; for Saved files it also tells how many are In
+  COLD_STORAGE. The Quasar counts after them are of all the files,
+  missing or not. It is followed by one line for every file missing or
+  not backed up yet; the line of a missing file also tells its Quasar
+  backup, or that it is not backed up.
 
   -XC or -CrossCopy, used with -AW or -AS to fill gaps in storage:
   copies files from Glade disk to bOreas (or vice versa) wherever

{rda_python_dsarch-3.0.13 → rda_python_dsarch-3.0.14}/src/rda_python_dsarch/pg_arch.py

@@ -1462,11 +1462,11 @@ class PgArch(PgOPT, PgCMD, PgSplit):
    # are not stored by GDEX and are checked neither on storage nor on Quasar
    # 'M' is the total of files missing, each counted once even when missing in both
    # places; a saved file found in COLD_STORAGE is not missing
-   # 'Q' are the missing files that can still be restored from a finished Quasar backup,
-   # so 'M' minus 'Q' is what is lost
-   QMSTATS = (('M', 'Missing'), ('Q', 'Missing but on Quasar'), ('G', 'Missing on Glade'),
-              ('O', 'Missing on Boreas'), ('C', 'In COLD_STORAGE'),
-              ('X', 'Not Checked (CGD/Remote)'))
+   # each 'Missing on' count is broken down by the Quasar status of its files, so
+   # it tells how many of them can still be restored and how many are lost; the
+   # Quasar counts that follow are of all the files, missing or not
+   QMSTATS = (('G', 'Missing on Glade'), ('O', 'Missing on Boreas'),
+              ('C', 'In COLD_STORAGE'), ('X', 'Not Checked (CGD/Remote)'))
 
    def check_file_storage(self, dsid, head, gidxs, wn, ws):
       """
@@ -1527,6 +1527,8 @@ class PgArch(PgOPT, PgCMD, PgSplit):
       total = [0, 0]
       bcnts = {k: [0, 0] for (k, m) in self.QBSTATS}
       mcnts = {k: [0, 0] for (k, m) in self.QMSTATS}
+      # Quasar status (and COLD_STORAGE for saved files) of the files missing per location
+      mbcnts = {'G': {}, 'O': {}}
       lines = []
       for i in range(fcnt):
          fname = frecs['fname'][i]
@@ -1555,7 +1557,6 @@ class PgArch(PgOPT, PgCMD, PgSplit):
             if bkey != 'N': bmsg += " (Backup ID {})".format(frecs['bid'][i])
             # physical copies
             msgs = []
-            missed = False
             if locflag != 'O':
                if cate == 'S':
                   lfile = "{}/{}/{}".format(self.RTPATH['SP'], ftype, fname)
@@ -1570,24 +1571,18 @@ class PgArch(PgOPT, PgCMD, PgSplit):
                if not op.exists(lfile):
                   mcnts['G'][0] += 1
                   mcnts['G'][1] += fsize
-                  msg = self.QMSTATS[2][1]
+                  self.add_file_count(mbcnts['G'], bkey, fsize)
+                  msg = self.QMSTATS[0][1]
                   if cfile:
+                     self.add_file_count(mbcnts['G'], 'C', fsize)
                      msg += ", Found in COLD_STORAGE " + cfile
-                  else:
-                     missed = True
                   msgs.append(msg)
             if locflag in ('O', 'B'):
                if not self.object_file_exists(dsid, fname, bucket):
                   mcnts['O'][0] += 1
                   mcnts['O'][1] += fsize
-                  msgs.append(self.QMSTATS[3][1])
-                  missed = True
-            if missed:
-               mcnts['M'][0] += 1
-               mcnts['M'][1] += fsize
-               if bkey in ('B', 'D'):
-                  mcnts['Q'][0] += 1
-                  mcnts['Q'][1] += fsize
+                  self.add_file_count(mbcnts['O'], bkey, fsize)
+                  msgs.append(self.QMSTATS[1][1])
             # one line per file: where it is missing and where it is backed up, so a
             # missing file shows whether it can be restored; a present file is listed
             # only when it is not backed up yet
@@ -1598,8 +1593,14 @@ class PgArch(PgOPT, PgCMD, PgSplit):
             if msgs: lines.append("{}: {}".format(finfo, '; '.join(msgs)))
       msg = "{} {} Files: Total {}".format(head, cname, self.file_number_size(total, wn, ws))
       for (k, m) in self.QMSTATS:
-         # the total missing is always shown, so a clean check reads 'Missing 0'
-         if mcnts[k][0] or k == 'M': msg += ", {} {}".format(m, self.file_number_size(mcnts[k], wn, ws))
+         if not mcnts[k][0]: continue
+         msg += ", {} {}".format(m, self.file_number_size(mcnts[k], wn, ws))
+         if k in mbcnts:
+            # e.g. Missing on Glade 6309(65.73GB) [Quasar Backup 10(1.20GB), Not Backed Up 6299(64.53GB)]
+            subs = [(bk, bm) for (bk, bm) in self.QBSTATS] + [('C', self.QMSTATS[2][1])]
+            parts = ["{} {}".format(bm, self.file_number_size(mbcnts[k][bk], wn, ws))
+                     for (bk, bm) in subs if bk in mbcnts[k]]
+            msg += " [{}]".format(', '.join(parts))
       for (k, m) in self.QBSTATS:
          if bcnts[k][0]: msg += ", {} {}".format(m, self.file_number_size(bcnts[k], wn, ws))
       self.OUTPUT.write(msg + "\n")
@@ -1607,10 +1608,17 @@ class PgArch(PgOPT, PgCMD, PgSplit):
          self.OUTPUT.write(line + "\n")
 
    @staticmethod
-   def file_number_size(cnts, wn, ws):
-      """Format [count, size] as 'count', 'size' or 'count(size)' per -WN/-WS."""
-      if wn and ws: return "{}({})".format(cnts[0], cnts[1])
-      return str(cnts[1] if ws else cnts[0])
+   def add_file_count(cnts, key, fsize):
+      """Add one file of fsize bytes to the [count, size] of key in dict cnts."""
+      if key not in cnts: cnts[key] = [0, 0]
+      cnts[key][0] += 1
+      cnts[key][1] += fsize
+
+   def file_number_size(self, cnts, wn, ws):
+      """Format [count, size] as 'count', 'size' or 'count(size)' per -WN/-WS, size readable."""
+      size = self.format_float_value(cnts[1])
+      if wn and ws: return "{}({})".format(cnts[0], size)
+      return size if ws else str(cnts[0])
 
    def object_file_exists(self, dsid, fname, bucket):
       """

{rda_python_dsarch-3.0.13 → rda_python_dsarch-3.0.14/src/rda_python_dsarch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 3.0.13
+Version: 3.0.14
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch