rda-python-dsarch 3.0.12__tar.gz → 3.0.14__tar.gz

{rda_python_dsarch-3.0.12/src/rda_python_dsarch.egg-info → rda_python_dsarch-3.0.14}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 3.0.12
+Version: 3.0.14
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch

{rda_python_dsarch-3.0.12 → rda_python_dsarch-3.0.14}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsarch"
-version = "3.0.12"
+version = "3.0.14"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsarch-3.0.12 → rda_python_dsarch-3.0.14}/src/rda_python_dsarch/dsarch.py

@@ -2049,7 +2049,7 @@ class DsArch(PgArch, PgMeta):
          self.print_row_format(pgrec, fnames, hash)
       self.view_keyvalues(dsid, kvalues, getkeys)
       if fnames and 'PE' in self.params: self.get_period_info()
-      if 'WN' in self.params: self.view_filenumber(dsid, 0)
+      if 'WN' in self.params or 'WS' in self.params: self.view_filenumber(dsid, 0)
       return 1   # get one dataset record
 
    def get_period_info(self):
@@ -2206,7 +2206,7 @@ class DsArch(PgArch, PgMeta):
       onames = self.params['ON'] if 'ON' in self.params else "I"
       qnames = fnames
       if fnames.find('Y') > -1 and fnames.find('X') < 0: qnames += 'X'
-      if 'WN' in self.params and fnames.find('I') < 0: qnames += 'I'
+      if ('WN' in self.params or 'WS' in self.params) and fnames.find('I') < 0: qnames += 'I'
       qnames += self.append_order_fields(onames, qnames, tname, "Y")
       condition = self.get_condition(tname)
       if 'ON' in self.params and ('OB' in self.params or re.search(r'Y', onames, re.I)):
@@ -2227,7 +2227,7 @@ class DsArch(PgArch, PgMeta):
          cnt = self.print_column_format(pgrecs, fnames, hash, lens)
          s = 's' if cnt > 1 else ''
          self.pglog("{} group record{} retrieved".format(cnt, s), self.LOGWRN)
-         if 'WN' in self.params: self.view_filenumber(dsid, pgrecs['gindex'], cnt)
+         if 'WN' in self.params or 'WS' in self.params: self.view_filenumber(dsid, pgrecs['gindex'], cnt)
       else:
          self.pglog("No group found for " + condition, self.LOGWRN)
 

{rda_python_dsarch-3.0.12 → rda_python_dsarch-3.0.14}/src/rda_python_dsarch/dsarch.usg

@@ -609,10 +609,11 @@ QSCAT observations.<:>
   Results can be filtered by group index (-GI), name (-GN), or parent
   index (-PI). The -GN option accepts the '%' wildcard.
 
-  For example, to retrieve the default group fields and file counts for
-  groups 1 and 2 of d260001:
+  For example, to retrieve the default group fields, the file numbers
+  and sizes, and the storage and Quasar backup checks for groups 1 and
+  2 of d260001:
 
-  dsarch d260001 GG -WN -GI 1 2
+  dsarch d260001 GG -WN -WS -GI 1 2
 
 <<Content of the output>>
 Dataset<=>d260001
@@ -621,7 +622,13 @@ GroupIndex<:>GroupName<:>ParentIndex<:>Title<:>GroupPattern<:>BackupFlag<:>Saved
 2<:>MONTHLY<:>0<:>MONTHLY OAFlux Products<:><:>P<:><:>monthly<:>
 2 group records retrieved
 d260001-G1: DS-11094192261 DC-327 WC-327 SS-8909173908 SC-51
+d260001-G1 Web Files: Total 327(11.09GB), Quasar Backup&DR 327(11.09GB)
+d260001-G1 Saved Files: Total 51(8.91GB), Quasar Backup&DR 51(8.91GB)
 d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
+d260001-G2 Web Files: Total 522(552.78MB), Missing on Glade 2(2.12MB) [Quasar Backup&DR 1(1.06MB), Not Backed Up 1(1.06MB)], Quasar Backup&DR 521(551.72MB), Not Backed Up 1(1.06MB)
+d260001-G2-W-D: monthly/2025/oaflux_202511.nc: Missing on Glade; on Quasar Backup&DR (Backup ID 3456789)
+d260001-G2-W-D: monthly/2025/oaflux_202512.nc: Missing on Glade; Not Backed Up
+d260001-G2 Saved Files: Total 51(465.79MB), Quasar Backup&DR 51(465.79MB)
 
 
 3.3.3 Delete Group Information
@@ -1909,12 +1916,41 @@ behavior.
 
   -WN or -WithFileNumber, recomputes and updates Saved/Web file
   counts for the dataset and any specified groups. With -GD
-  (-GetDataset), the counts are calculated on the fly and included
-  in the output.
+  (-GetDataset) or -GG (-GetGroup), shows the Web/Saved file numbers
+  of the dataset or each group, and checks their files as described
+  below.
 
   -WS or -WithFileSize, rechecks Web file sizes and timestamps for
   provided web files with -WF (-WebFile), and updates them into GDEXDB
-  This Mode option works with action -SW (SetWebFile) only.
+  with action -SW (SetWebFile). With -GD (-GetDataset) or -GG
+  (-GetGroup), shows the Web/Saved file sizes of the dataset or each
+  group, and checks their files as described below.
+
+  With -GD/-GG, either -WN or -WS also checks every Web and Saved file
+  (including the files of sub groups for a group) that is not deleted:
+    - a file with Location Flag G or B must exist on Glade, and one
+      with Location Flag O or B on Boreas (object store);
+    - a Saved file on Glade (Location Flag G or B) is looked for in
+      COLD_STORAGE, <decsdata>/COLD_STORAGE/<dsid> or
+      <decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE/<dsid>, and is
+      counted as In COLD_STORAGE whether or not it is on Glade too;
+    - its Quasar backup, by the backup file (bfile) of its Backup ID:
+      type B for Quasar Backup, type D for Quasar Backup & Disaster
+      Recovery, counted once the backup file status is A; Pending
+      when it is not, and Not Backed Up when it has no Backup ID.
+  Files with Location Flag C (CGD) or R (remote) are not stored by
+  GDEX and are not checked. One summary line per file category shows
+  the file counts with -WN and the total file sizes with -WS (both as
+  Count(Size) when both options are given, sizes in KB/MB/GB/TB): the
+  Total, then the result of each check. Each Missing on Glade/Boreas
+  count is followed by the Quasar status of its files in [], e.g.
+  [Quasar Backup&DR 1(1.06MB), Not Backed Up 1(1.06MB)], so it tells
+  how many of the missing files can be restored with action -RQ and
+  how many are lost; for Saved files it also tells how many are In
+  COLD_STORAGE. The Quasar counts after them are of all the files,
+  missing or not. It is followed by one line for every file missing or
+  not backed up yet; the line of a missing file also tells its Quasar
+  backup, or that it is not backed up.
 
   -XC or -CrossCopy, used with -AW or -AS to fill gaps in storage:
   copies files from Glade disk to bOreas (or vice versa) wherever

{rda_python_dsarch-3.0.12 → rda_python_dsarch-3.0.14}/src/rda_python_dsarch/pg_arch.py

@@ -21,6 +21,7 @@ and caching dataset/group metadata.
 import time
 import re
 import os
+import glob
 from os import path as op
 from rda_python_common.pg_split import PgSplit
 from rda_python_common.pg_cmd import PgCMD
@@ -61,6 +62,8 @@ class PgArch(PgOPT, PgCMD, PgSplit):
          'SP' : '',         # dataset savedpath
       }
       self.webpaths = {}
+      self.OBJKEYS = {}    # object keys listed per bucket:dsid, by object_file_exists()
+      self.COLDDIRS = {}   # cold storage directories found per dsid, by cold_storage_file()
       self.savedpaths = {}
       self.grouptypes = {}
       self.CORDERS = {}
@@ -1417,27 +1420,256 @@ class PgArch(PgOPT, PgCMD, PgSplit):
       else:
          cnt = 1
          grecs = [gindex]
+      # -WN shows the file numbers and -WS the file sizes; either one also checks the
+      # files on storage and on Quasar, see check_file_storage()
+      wn = 'WN' in self.params
+      ws = 'WS' in self.params
       fields = ("dwebcnt, webcnt, cdwcnt, nwebcnt, dweb_size, nweb_size, savedcnt, saved_size")
+      dcnd = "dsid = '{}'".format(dsid)
       for i in range(cnt):
          if grecs[i] == 0:
             table = "dataset"
-            cnd = "dsid = '{}'".format(dsid)
+            cnd = dcnd
             head = dsid
          else:
             table = "dsgroup"
-            cnd = "dsid = '{}' AND gindex = {}".format(dsid, grecs[i])
+            cnd = "{} AND gindex = {}".format(dcnd, grecs[i])
             head = "{}-G{}".format(dsid, grecs[i])
          pgrec = self.pgget(table, fields, cnd, self.PGOPT['extlog'])
          if pgrec:
             msg = ''
             if pgrec['webcnt']:
-               if pgrec['dwebcnt']: msg += " DS-{} DC-{}".format(pgrec['dweb_size'], pgrec['dwebcnt'])
-               if pgrec['cdwcnt']: msg += " CDC-{}".format(pgrec['cdwcnt'])
-               if pgrec['nwebcnt']: msg += " NS-{} NC-{}".format(pgrec['nweb_size'], pgrec['nwebcnt'])
-               msg += " WC-{}".format(pgrec['webcnt'])
+               if pgrec['dwebcnt']:
+                  if ws: msg += " DS-{}".format(pgrec['dweb_size'])
+                  if wn: msg += " DC-{}".format(pgrec['dwebcnt'])
+               if pgrec['cdwcnt'] and wn: msg += " CDC-{}".format(pgrec['cdwcnt'])
+               if pgrec['nwebcnt']:
+                  if ws: msg += " NS-{}".format(pgrec['nweb_size'])
+                  if wn: msg += " NC-{}".format(pgrec['nwebcnt'])
+               if wn: msg += " WC-{}".format(pgrec['webcnt'])
             if pgrec['savedcnt']:
-               msg += " SS-{} SC-{}".format(pgrec['saved_size'], pgrec['savedcnt'])
-            if msg: print("{}:{}".format(head, msg))
+               if ws: msg += " SS-{}".format(pgrec['saved_size'])
+               if wn: msg += " SC-{}".format(pgrec['savedcnt'])
+            if msg: self.OUTPUT.write("{}:{}\n".format(head, msg))
+         # group counts include the sub groups, so check the files of all of them
+         gidxs = self.get_all_subgroups(dcnd, grecs[i]) if grecs[i] else None
+         self.check_file_storage(dsid, head, gidxs, wn, ws)
+
+   # Quasar backup status of a file, by bfile.type and bfile.status of its bid
+   QBSTATS = (('B', 'Quasar Backup'), ('D', 'Quasar Backup&DR'),
+              ('P', 'Quasar Backup Pending'), ('N', 'Not Backed Up'))
+   # where a missing file was looked for, by locflag; 'C' (CGD) and 'R' (remote) files
+   # are not stored by GDEX and are checked neither on storage nor on Quasar
+   # 'M' is the total of files missing, each counted once even when missing in both
+   # places; a saved file found in COLD_STORAGE is not missing
+   # each 'Missing on' count is broken down by the Quasar status of its files, so
+   # it tells how many of them can still be restored and how many are lost; the
+   # Quasar counts that follow are of all the files, missing or not
+   QMSTATS = (('G', 'Missing on Glade'), ('O', 'Missing on Boreas'),
+              ('C', 'In COLD_STORAGE'), ('X', 'Not Checked (CGD/Remote)'))
+
+   def check_file_storage(self, dsid, head, gidxs, wn, ws):
+      """
+      Check the web and saved files of a dataset or group on storage and Quasar.
+
+      For every web/saved file that is not deleted: a Glade copy (locflag G or B)
+      must exist under the dataset home, a Boreas copy (locflag O or B) must exist
+      in the object store, and a Glade saved file is looked for under COLD_STORAGE,
+      counted there whether or not it is still on Glade too. The Quasar backup status comes from bfile.type ('B' backup,
+      'D' backup and disaster recovery) of the file's bid, counted only once the
+      backup file status is 'A'. Writes the totals and every missing or not
+      backed up file to self.OUTPUT.
+
+      Args:
+         dsid  -- dataset ID string
+         head  -- line prefix, dsid or dsid-G<gindex>
+         gidxs -- group indices to check, None for the whole dataset
+         wn    -- show file numbers (-WN)
+         ws    -- show file sizes (-WS)
+      """
+      if gidxs is None:
+         gcnd = ''
+      elif len(gidxs) == 1:
+         gcnd = " AND {{0}}.gindex = {}".format(gidxs[0])
+      else:
+         gcnd = " AND {{0}}.gindex IN ({})".format(', '.join(str(g) for g in gidxs))
+      bflds = "bfile.type btype, bfile.status bstat"
+      bjoin = "LEFT JOIN bfile ON {0}.bid = bfile.bid"
+      wrecs = self.pgmget_wfile_join(dsid, bjoin.format('wfile'),
+                                     "wfile.wid fid, wfile.wfile fname, wfile.type, wfile.data_size, " +
+                                     "wfile.locflag, wfile.bid, " + bflds,
+                                     "wfile.status <> 'D'" + gcnd.format('wfile'), self.PGOPT['extlog'])
+      srecs = self.pgmget("sfile " + bjoin.format('sfile'),
+                          "sfile.sid fid, sfile.sfile fname, sfile.type, sfile.data_size, " +
+                          "sfile.locflag, sfile.bid, " + bflds,
+                          "sfile.dsid = '{}' AND sfile.status <> 'D'".format(dsid) + gcnd.format('sfile'),
+                          self.PGOPT['extlog'])
+      for (cate, frecs) in (('W', wrecs), ('S', srecs)):
+         fcnt = len(frecs['fid']) if frecs else 0
+         if fcnt: self.view_file_storage(dsid, head, cate, frecs, fcnt, wn, ws)
+
+   def view_file_storage(self, dsid, head, cate, frecs, fcnt, wn, ws):
+      """
+      Check one category (W-web or S-saved) of file records and write the report.
+
+      Args:
+         dsid  -- dataset ID string
+         head  -- line prefix, dsid or dsid-G<gindex>
+         cate  -- 'W' for web files, 'S' for saved files
+         frecs -- multi-record dict from check_file_storage()
+         fcnt  -- number of records in frecs
+         wn    -- show file numbers (-WN)
+         ws    -- show file sizes (-WS)
+      """
+      self.cache_rtpath(dsid)
+      bucket = 'gdex-decsdata' if cate == 'S' else self.PGLOG['OBJCTBKT']
+      cname = 'Saved' if cate == 'S' else 'Web'
+      total = [0, 0]
+      bcnts = {k: [0, 0] for (k, m) in self.QBSTATS}
+      mcnts = {k: [0, 0] for (k, m) in self.QMSTATS}
+      # Quasar status (and COLD_STORAGE for saved files) of the files missing per location
+      mbcnts = {'G': {}, 'O': {}}
+      lines = []
+      for i in range(fcnt):
+         fname = frecs['fname'][i]
+         ftype = frecs['type'][i]
+         fsize = frecs['data_size'][i] or 0
+         locflag = frecs['locflag'][i]
+         finfo = "{}-{}-{}: {}".format(head, cate, ftype, fname)
+         total[0] += 1
+         total[1] += fsize
+         if locflag in ('C', 'R'):
+            # not stored by GDEX, so neither on storage nor on Quasar
+            mcnts['X'][0] += 1
+            mcnts['X'][1] += fsize
+            continue
+         else:
+            # quasar backup, by the backup file of the bid
+            if not frecs['bid'][i]:
+               bkey = 'N'
+            elif frecs['bstat'][i] == 'A' and frecs['btype'][i] in ('B', 'D'):
+               bkey = frecs['btype'][i]
+            else:
+               bkey = 'P'
+            bcnts[bkey][0] += 1
+            bcnts[bkey][1] += fsize
+            bmsg = dict(self.QBSTATS)[bkey]
+            if bkey != 'N': bmsg += " (Backup ID {})".format(frecs['bid'][i])
+            # physical copies
+            msgs = []
+            if locflag != 'O':
+               if cate == 'S':
+                  lfile = "{}/{}/{}".format(self.RTPATH['SP'], ftype, fname)
+               else:
+                  lfile = self.join_paths(self.RTPATH.get(ftype, self.RTPATH['D']), fname)
+               # every Glade saved file is looked for in cold storage, so the count tells
+               # how much of the saved data is on tape, whether or not it is on Glade too
+               cfile = self.cold_storage_file(dsid, ftype, fname) if cate == 'S' else None
+               if cfile:
+                  mcnts['C'][0] += 1
+                  mcnts['C'][1] += fsize
+               if not op.exists(lfile):
+                  mcnts['G'][0] += 1
+                  mcnts['G'][1] += fsize
+                  self.add_file_count(mbcnts['G'], bkey, fsize)
+                  msg = self.QMSTATS[0][1]
+                  if cfile:
+                     self.add_file_count(mbcnts['G'], 'C', fsize)
+                     msg += ", Found in COLD_STORAGE " + cfile
+                  msgs.append(msg)
+            if locflag in ('O', 'B'):
+               if not self.object_file_exists(dsid, fname, bucket):
+                  mcnts['O'][0] += 1
+                  mcnts['O'][1] += fsize
+                  self.add_file_count(mbcnts['O'], bkey, fsize)
+                  msgs.append(self.QMSTATS[1][1])
+            # one line per file: where it is missing and where it is backed up, so a
+            # missing file shows whether it can be restored; a present file is listed
+            # only when it is not backed up yet
+            if msgs:
+               msgs.append("on " + bmsg if bkey in ('B', 'D') else bmsg)
+            elif bkey in ('N', 'P'):
+               msgs.append(bmsg)
+            if msgs: lines.append("{}: {}".format(finfo, '; '.join(msgs)))
+      msg = "{} {} Files: Total {}".format(head, cname, self.file_number_size(total, wn, ws))
+      for (k, m) in self.QMSTATS:
+         if not mcnts[k][0]: continue
+         msg += ", {} {}".format(m, self.file_number_size(mcnts[k], wn, ws))
+         if k in mbcnts:
+            # e.g. Missing on Glade 6309(65.73GB) [Quasar Backup 10(1.20GB), Not Backed Up 6299(64.53GB)]
+            subs = [(bk, bm) for (bk, bm) in self.QBSTATS] + [('C', self.QMSTATS[2][1])]
+            parts = ["{} {}".format(bm, self.file_number_size(mbcnts[k][bk], wn, ws))
+                     for (bk, bm) in subs if bk in mbcnts[k]]
+            msg += " [{}]".format(', '.join(parts))
+      for (k, m) in self.QBSTATS:
+         if bcnts[k][0]: msg += ", {} {}".format(m, self.file_number_size(bcnts[k], wn, ws))
+      self.OUTPUT.write(msg + "\n")
+      for line in lines:
+         self.OUTPUT.write(line + "\n")
+
+   @staticmethod
+   def add_file_count(cnts, key, fsize):
+      """Add one file of fsize bytes to the [count, size] of key in dict cnts."""
+      if key not in cnts: cnts[key] = [0, 0]
+      cnts[key][0] += 1
+      cnts[key][1] += fsize
+
+   def file_number_size(self, cnts, wn, ws):
+      """Format [count, size] as 'count', 'size' or 'count(size)' per -WN/-WS, size readable."""
+      size = self.format_float_value(cnts[1])
+      if wn and ws: return "{}({})".format(cnts[0], size)
+      return size if ws else str(cnts[0])
+
+   def object_file_exists(self, dsid, fname, bucket):
+      """
+      Check whether an object key, or a directory key such as a zarr store, exists.
+
+      Lists the whole dataset prefix once per dataset and bucket, so checking a
+      dataset of many object files costs one object store call instead of one per
+      file. A listing error is reported once and the files are then taken as present.
+      """
+      ckey = "{}:{}".format(bucket, dsid)
+      if ckey not in self.OBJKEYS:
+         keys = set()
+         flist = self.object_glob(dsid + '/', bucket, 0, self.PGOPT['wrnlog'])
+         if flist == self.FAILURE:
+            keys = None
+         elif flist:
+            for key in flist:
+               # a key under a directory key also stands for each of its parent directories
+               while key and key not in keys:
+                  keys.add(key)
+                  key = op.dirname(key)
+         self.OBJKEYS[ckey] = keys
+      keys = self.OBJKEYS[ckey]
+      if keys is None: return True
+      return self.join_paths(dsid, fname) in keys
+
+   def cold_storage_file(self, dsid, stype, sfile):
+      """
+      Find a saved file moved into GLADE cold storage.
+
+      A dataset is moved either to <DECSHOME>/COLD_STORAGE/<dsid> or to
+      <DECSHOME>/cold_storage_<YYYYMMDD>/COLD_STORAGE/<dsid>; both are searched,
+      the root one first and then the dated ones newest first. Checking a migrated
+      file only reads its inode, it does not recall it from tape.
+
+      Returns:
+         Path of the file in cold storage, or None if not found.
+      """
+      if dsid not in self.COLDDIRS:
+         dhome = self.PGLOG['DECSHOME']
+         cdirs = []
+         for cname in ('COLD_STORAGE', 'COLD_STORAGE_SINGLE_COPY'):
+            cdirs.append("{}/{}/{}".format(dhome, cname, dsid))
+         for cdir in sorted(glob.glob(dhome + "/cold_storage_*"), reverse = True):
+            for cname in ('COLD_STORAGE', 'COLD_STORAGE_SINGLE_COPY'):
+               cdirs.append("{}/{}/{}".format(cdir, cname, dsid))
+         self.COLDDIRS[dsid] = [cdir for cdir in cdirs if op.isdir(cdir)]
+      for cdir in self.COLDDIRS[dsid]:
+         cfile = "{}/{}/{}".format(cdir, stype, sfile)
+         if op.exists(cfile): return cfile
+      return None
    
    def get_bid_numbers(self, bfiles):
       """

{rda_python_dsarch-3.0.12 → rda_python_dsarch-3.0.14/src/rda_python_dsarch.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 3.0.12
+Version: 3.0.14
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch