rda-python-dsarch 3.0.7__py3-none-any.whl → 3.0.9__py3-none-any.whl

rda_python_dsarch/dsarch.py

@@ -45,6 +45,7 @@ class DsArch(PgArch, PgMeta):
       MODCNT   -- number of existing records modified during the current action
       OVERRIDE -- non-zero when the -OE (OverrideExist) flag is active
       TARFILES -- dict mapping tar archive paths to their member-file metadata
+      BCKNOTES -- dict mapping backup IDs to their bfile.note member/move history
       VINDEX   -- dict mapping version indices to version records
       CHGGRPS  -- dict of group indices whose file-number counts need updating
       ERRMSG   -- accumulated error message string for the current action
@@ -54,6 +55,7 @@ class DsArch(PgArch, PgMeta):
       super().__init__()  # initialize parent class
       self.ERRCNT = self.RETSTAT = self.ALLCNT = self.ADDCNT = self.MODCNT = self.OVERRIDE = 0
       self.TARFILES = {}
+      self.BCKNOTES = {}
       self.VINDEX = {}
       self.CHGGRPS = {}
       self.ERRMSG = ''
@@ -1616,7 +1618,7 @@ class DsArch(PgArch, PgMeta):
          (bfile, qfile) = self.get_backup_filenames(self.params['QF'][i], dsid)
          pgrec = self.pgget(tname, "*", "bfile = '{}'".format(bfile), self.LGEREX)
          if not pgrec:
-            self.pglog("Backup-{}: is not in RDADB".format(bfile, self.PGLOG['MISSFILE']), self.LOGWRN)
+            self.pglog("Backup-{}: is not in RDADB".format(bfile), self.LOGWRN)
             continue
          ms = re.match(r'^<([a-z]\d{6}|ds\d+\.\d+)_(\w)_\d+\.txt>', pgrec['note'])
          if not ms:
@@ -1625,7 +1627,11 @@ class DsArch(PgArch, PgMeta):
          ftype = ms.group(2)
          if fdsid != dsid: qfile = "/{}/{}".format(fdsid, bfile)
          info = self.check_backup_file(qfile)
-         if not info:
+         if info == self.FAILURE:   # error checking the endpoint, not a missing file
+            errmsg = "Backup-{}: Error checking file on Quasar Endpoint {}".format(qfile, endpoint)
+            if self.PGLOG['SYSERR']: errmsg += "\n" + self.PGLOG['SYSERR']
+            self.pglog(errmsg, self.PGOPT['extlog'])
+         elif not info:
             self.pglog("Backup-{}: {}".format(qfile, self.PGLOG['MISSFILE']), self.LOGWRN)
             continue
          endpath = 'decsdata' if ftype == 'S' else 'data'
@@ -1635,6 +1641,7 @@ class DsArch(PgArch, PgMeta):
          tofile = "{}/{}".format(todir, op.basename(bfile))
          tarfile = "{}/{}".format(tardir, op.basename(bfile))
          self.TARFILES[pgrec['bid']] = tarfile
+         self.BCKNOTES[pgrec['bid']] = pgrec['note']
          tinfo = self.check_local_file(tarfile, 0, self.PGOPT['extlog'])
          if tinfo:
             self.pglog(tarfile + ": tar file exists already", self.PGOPT['wrnlog'])
@@ -1644,7 +1651,7 @@ class DsArch(PgArch, PgMeta):
             self.set_local_mode(tarfile, 1, 0o664)
       if self.PGLOG['DSCHECK']:
          self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
-      self.pglog("{} of Quasar Backup file{} downloaded for {}".format(dcnt, self.ALLCNT, s, dsid), self.LOGWRN)
+      self.pglog("{} of {} Quasar Backup file{} downloaded for {}".format(dcnt, self.ALLCNT, s, dsid), self.LOGWRN)
       if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
 
    def restore_backup_webfiles(self):
@@ -1699,6 +1706,9 @@ class DsArch(PgArch, PgMeta):
          if ainfo:
             self.pglog(afile + ": File exists already", self.PGOPT['wrnlog'])
          else:
+            if pgrec['bid'] not in self.TARFILES:
+               self.pglog("{}: Backup tar file of Backup Id {} is not downloaded".format(wfile, pgrec['bid']), self.PGOPT['wrnlog'])
+               continue
             tarfile = self.TARFILES[pgrec['bid']]
             ainfo = self.get_backup_member_file(pgrec, tarfile, tardir)
             tarcmd = "tar -xvf {} -C {} {}".format(tarfile, tardir, wfile)
@@ -1706,6 +1716,7 @@ class DsArch(PgArch, PgMeta):
             ainfo = self.check_local_file(afile, 0, self.PGOPT['extlog'])
             if not ainfo:
                self.pglog("{}: Error untar File {}".format(tarfile, afile), self.PGOPT['wrnlog'])
+               continue
             if warch: wcnt += 1
          if ainfo['data_size'] != pgrec['data_size']:
             self.pglog("{}: Different Restored/RDADB file sizes {}/{}".format(afile, ainfo['data_size'], pgrec['data_size']), self.PGOPT['wrnlog'])
@@ -1735,9 +1746,10 @@ class DsArch(PgArch, PgMeta):
       Returns:
          Path to the extracted file, or None on failure.
       """
-      mfile = wfile = pgrec['wfile']
-      note = pgrec['note']
-      while not re.search(r'{}<:>'.format(mfile), note):
+      mfile = wfile = pgrec['wfile'] if 'wfile' in pgrec else pgrec['sfile']
+      # the member list and move history are on bfile.note, not on the wfile/sfile record
+      note = self.BCKNOTES.get(pgrec['bid'])
+      while note and not re.search(r'{}<:>'.format(mfile), note):
          ms = re.search(r'>MV .* File (\S+) To .*{}'.format(mfile), note)
          if not ms: break
          mfile = ms.group(1)
@@ -1801,6 +1813,9 @@ class DsArch(PgArch, PgMeta):
          if ainfo:
             self.pglog(afile + ": File exists already", self.PGOPT['wrnlog'])
          else:
+            if pgrec['bid'] not in self.TARFILES:
+               self.pglog("{}: Backup tar file of Backup Id {} is not downloaded".format(sfile, pgrec['bid']), self.PGOPT['wrnlog'])
+               continue
             tarfile = self.TARFILES[pgrec['bid']]
             ainfo = self.get_backup_member_file(pgrec, tarfile, tardir)
             tarcmd = "tar -xvf {} -C {} {}".format(tarfile, tardir, sfile)
@@ -1808,6 +1823,7 @@ class DsArch(PgArch, PgMeta):
             ainfo = self.check_local_file(afile, 0, self.PGOPT['extlog'])
             if not ainfo:
                self.pglog("{}: Error untar File {}".format(tarfile, afile), self.PGOPT['wrnlog'])
+               continue
             if sarch: scnt += 1
          if ainfo['data_size'] != pgrec['data_size']:
             self.pglog("{}: Different Restored/RDADB file sizes {}/{}".format(afile, ainfo['data_size'], pgrec['data_size']), self.PGOPT['wrnlog'])

{rda_python_dsarch-3.0.7.dist-info → rda_python_dsarch-3.0.9.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 3.0.7
+Version: 3.0.9
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch

{rda_python_dsarch-3.0.7.dist-info → rda_python_dsarch-3.0.9.dist-info}/RECORD

@@ -2,13 +2,13 @@ rda_python_dsarch/PgArch.py,sha256=_hfBFyft9rSC35QBIbtQRkF3TQE7tuT7Dz-B-KmVsVU,5
 rda_python_dsarch/PgMeta.py,sha256=FV4GytgnJK00FUlyM6iHqpNd7-XTVHO1j5tZtcLTWgY,26845
 rda_python_dsarch/__init__.py,sha256=-ympZMoYfvP0lxSq06GWQemQlj4HwV9tm1JNxKpSU90,812
 rda_python_dsarch/ds_arch.py,sha256=tdlelcLMmlffJjIxEvd-xlI0gwDSuDdKO1hX2-oAOcs,233525
-rda_python_dsarch/dsarch.py,sha256=s788Mmbjkq0ox2eNb5Rmantcj8W_WnKr3veO8m4KbWQ,254896
+rda_python_dsarch/dsarch.py,sha256=sUTHmPUCm7YHqTe103LR6MfAAVFO3xmFbmp_jBzUAyA,255970
 rda_python_dsarch/dsarch.usg,sha256=LpteXfUe9RBSbmM3HputbtZjEEyUSKnYvOfphH0UQrA,106590
 rda_python_dsarch/pg_arch.py,sha256=-HD-CynFzvcBsL9w6EN-4O5RqdXvspbRUbNzs6njSaQ,80494
 rda_python_dsarch/pg_meta.py,sha256=FFEkphGAT--66zvMNDPfiFG4nZQpXed_yVGMARdZb-o,46981
-rda_python_dsarch-3.0.7.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
-rda_python_dsarch-3.0.7.dist-info/METADATA,sha256=Z0RcGafKWHK_yHyGdnhxQQ_NBQ7Lyv66JStLsiJ2lv0,7165
-rda_python_dsarch-3.0.7.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
-rda_python_dsarch-3.0.7.dist-info/entry_points.txt,sha256=BA1iRAFIZExG0656Ri9f-_OKlSK31CuABWPQSsA9MsI,64
-rda_python_dsarch-3.0.7.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
-rda_python_dsarch-3.0.7.dist-info/RECORD,,
+rda_python_dsarch-3.0.9.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
+rda_python_dsarch-3.0.9.dist-info/METADATA,sha256=7NHcuDawOJjbn27tMyRh8yh0WEsGRNnh4rPQ4D7fli4,7165
+rda_python_dsarch-3.0.9.dist-info/WHEEL,sha256=YVMoNqKzERt-wjUZwJ33xBGAwnFl-4cqbYkTtWa4itE,91
+rda_python_dsarch-3.0.9.dist-info/entry_points.txt,sha256=BA1iRAFIZExG0656Ri9f-_OKlSK31CuABWPQSsA9MsI,64
+rda_python_dsarch-3.0.9.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
+rda_python_dsarch-3.0.9.dist-info/RECORD,,