rda-python-dsarch 2.0.1__py3-none-any.whl → 2.0.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -77,13 +77,8 @@ class DsArch(PgArch, PgMeta):
77
77
  elif self.PGOPT['ACTS'] == self.OPTS['AS'][0]:
78
78
  self.ALLCNT = len(self.params['SF']) if 'SF' in self.params else len(self.params['LF'])
79
79
  self.cache_group_info(self.ALLCNT)
80
- if 'XC' in self.params:
81
- self.crosscopy_saved_files('Copy')
82
- elif 'XM' in self.params:
83
- self.crosscopy_saved_files('Move')
84
- else:
85
- self.archive_saved_files()
86
- if 'CL' in self.params: self.delete_local_files()
80
+ self.archive_saved_files()
81
+ if 'CL' in self.params: self.delete_local_files()
87
82
  elif self.PGOPT['ACTS'] == self.OPTS['AW'][0]:
88
83
  self.ALLCNT = len(self.params['WF']) if 'WF' in self.params else len(self.params['LF'])
89
84
  if 'ML' in self.params and 'GX' in self.params: del self.params['ML']
@@ -306,7 +301,6 @@ class DsArch(PgArch, PgMeta):
306
301
  self.cmdlog(None, 0, self.LOGWRN|self.SNDEML)
307
302
  else:
308
303
  self.cmdlog()
309
- self.pgexit(0)
310
304
 
311
305
  # archive web/object files
312
306
  def archive_web_files(self):
@@ -727,13 +721,12 @@ class DsArch(PgArch, PgMeta):
727
721
  # archive save files
728
722
  def archive_saved_files(self):
729
723
  tname = 'sfile'
730
- dftloc = None
724
+ dftloc = 'G'
731
725
  dsid = self.params['DS']
732
726
  dcnd = "dsid = '{}'".format(dsid)
733
727
  s = 's' if self.ALLCNT > 1 else ''
734
728
  bidx = chksize = 0
735
729
  dflags = {}
736
- bucket = "gdex-decsdata" # object store bucket
737
730
  self.pglog("Archive {} Saved file{} of {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
738
731
  if 'ZD' in self.params or 'UZ' in self.params:
739
732
  self.compress_localfile_list(self.PGOPT['CACT'], self.ALLCNT)
@@ -753,18 +746,17 @@ class DsArch(PgArch, PgMeta):
753
746
  self.params['AF'] = [None]*self.ALLCNT
754
747
  self.OPTS['AF'][2] |= 2
755
748
  if 'LC' not in self.params:
756
- self.params['LC'] = [None]*self.ALLCNT
749
+ self.params['LC'] = [dftloc]*self.ALLCNT
757
750
  self.OPTS['LC'][2] |= 2
758
751
  if 'SZ' not in self.params:
759
752
  self.params['SZ'] = [0]*self.ALLCNT
760
753
  self.OPTS['SZ'][2] |= 2
761
754
  if 'MC' not in self.params: self.params['MC'] = [None]*self.ALLCNT
762
- reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt = ocnt = 0
755
+ reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt = 0
763
756
  perrcnt = self.ALLCNT
764
757
  efiles = [1]*self.ALLCNT
765
758
  if self.PGSIG['BPROC'] > 1:
766
759
  afiles = [None]*self.ALLCNT
767
- ofiles = [None]*self.ALLCNT
768
760
  lfiles = [None]*self.ALLCNT
769
761
  fnames = None
770
762
  override = self.OVERRIDE
@@ -790,19 +782,9 @@ class DsArch(PgArch, PgMeta):
790
782
  self.params['LF'][i] = self.params['SF'][i] = None
791
783
  continue
792
784
  locflag = self.params['LC'][i]
793
- if not locflag or locflag == 'R':
794
- if not dftloc: dftloc = self.get_dataset_locflag(dsid)
795
- locflag = self.params['LC'][i] = dftloc
796
785
  if locflag == 'C': self.pglog(lfile + ": Cannot Archive Saved File for CGD data", self.PGOPT['extlog'])
797
- if locflag == 'B':
798
- oarch = sarch = 1
799
- elif locflag == 'O':
800
- oarch = 1
801
- sarch = 0
802
- else:
803
- oarch = 0
804
- sarch = 1
805
- if oarch: self.pglog(lfile + ": Cannot Archive Saved File onto Boreas", self.PGOPT['extlog'])
786
+ locflag = self.params['LC'][i] = dftloc
787
+ sarch = 1
806
788
  if not self.params['MC'][i]: self.params['MC'][i] = self.get_md5sum(lfile) # re-get MD5 Checksum
807
789
  if not (self.params['SZ'][i] and self.params['SZ'][i] == lsize):
808
790
  self.params['SZ'][i] = lsize
@@ -818,9 +800,8 @@ class DsArch(PgArch, PgMeta):
818
800
  else:
819
801
  self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['extlog'])
820
802
  continue
821
- afile = (self.get_saved_path(i, sfile, 1, type) if sarch else None)
803
+ afile = self.get_saved_path(i, sfile, 1, type)
822
804
  sfile = self.get_saved_path(i, sfile, 0, type)
823
- ofile = (self.join_paths(dsid, sfile) if oarch else None)
824
805
  pgrec = self.pgget(tname, "*", "{} AND sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
825
806
  if not pgrec:
826
807
  pgrec = self.pgget(tname, "type", "{} AND sfile = '{}'".format(dcnd, sfile), self.PGOPT['extlog'])
@@ -852,45 +833,15 @@ class DsArch(PgArch, PgMeta):
852
833
  efiles[i] = 1
853
834
  dflags['G'] = self.PGLOG['DECSHOME']
854
835
  continue
855
- if oarch:
856
- replace = 0
857
- info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
858
- if info:
859
- if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
860
- self.pglog("Object-{}-{}: Same-Checksum ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['emllog'])
861
- oarch = 0
862
- elif info['data_size'] == lsize:
863
- self.pglog("Object-{}-{}: Same-Size ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['wrnlog'])
864
- oarch = 0
865
- elif vsnctl and not override:
866
- self.pglog("Object-{}-{}: Cannot rearchive version controlled file".format(bucket, ofile), self.PGOPT['extlog'])
867
- self.params['SF'][i] = None
868
- continue
869
- else:
870
- replace = 1
871
- elif info is not None:
836
+ if sarch:
837
+ if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
872
838
  errcnt += 1
873
839
  efiles[i] = 1
874
- dflags['O'] = bucket
875
840
  continue
876
- if (oarch + sarch) > 0:
877
- if sarch:
878
- if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
879
- errcnt += 1
880
- efiles[i] = 1
881
- continue
882
- acnt += 1
883
- if oarch:
884
- if replace: self.delete_object_file(ofile, bucket)
885
- if not self.local_copy_object(ofile, lfile, bucket, None, self.PGOPT['emerol']|override):
886
- errcnt += 1
887
- efiles[i] = 1
888
- continue
889
- ocnt += 1
841
+ acnt += 1
890
842
  if self.PGLOG['DSCHECK']: chksize += lsize
891
843
  if self.PGSIG['BPROC'] > 1:
892
844
  afiles[i] = afile
893
- ofiles[i] = ofile
894
845
  lfiles[i] = lfile
895
846
  bgcnt += 1
896
847
  if self.PGSIG['BPROC'] < 2:
@@ -898,12 +849,8 @@ class DsArch(PgArch, PgMeta):
898
849
  info = None
899
850
  if sarch:
900
851
  info = self.check_local_file(afile, 1, self.PGOPT['emerol']|self.PFSIZE)
901
- elif oarch:
902
- info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
903
852
  elif pgrec:
904
853
  info = self.get_file_origin_info(sfile, pgrec)
905
- elif locflag == 'O':
906
- info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
907
854
  sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
908
855
  if not sid:
909
856
  self.params['LF'][i] = self.params['SF'][i] = None
@@ -923,11 +870,8 @@ class DsArch(PgArch, PgMeta):
923
870
  self.check_background(None, 0, self.LOGWRN, 1)
924
871
  for i in range(self.ALLCNT):
925
872
  if afiles[i]: self.validate_gladearch(afiles[i], lfiles[i], i)
926
- if ofiles[i]: self.validate_objectarch(ofiles[i], lfiles[i], bucket, i)
927
873
  if acnt > 0:
928
874
  self.pglog("{} of {} Saved file{} archived for {}".format(acnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
929
- if ocnt > 0:
930
- self.pglog("{] of {} Object file{} archived for {}".format(ocnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
931
875
  if self.PGLOG['DSCHECK']:
932
876
  self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
933
877
  self.pglog("{}/{} of {} Saved file record{} added/modified for {}!".format(self.ADDCNT, self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
@@ -1231,147 +1175,6 @@ class DsArch(PgArch, PgMeta):
1231
1175
  self.reset_rdadb_version(dsid)
1232
1176
  if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
1233
1177
 
1234
- # cross copy save files between glade and object store
1235
- def crosscopy_saved_files(self, aname):
1236
- tname = 'sfile'
1237
- dsid = self.params['DS']
1238
- dcnd = "dsid = '{}'".format(dsid)
1239
- s = 's' if self.ALLCNT > 1 else ''
1240
- bidx = chksize = 0
1241
- dflags = {}
1242
- bucket = "gdex-decsdata"
1243
- self.pglog("Cross {} {} Saved file{} of {} ...".format(aname, self.ALLCNT, s, dsid), self.WARNLG)
1244
- self.validate_multiple_values(tname, self.ALLCNT)
1245
- if self.PGLOG['DSCHECK']:
1246
- bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
1247
- if bidx > 0:
1248
- self.pglog("{} of {} file{} processed for SAVED archive".format(bidx, self.ALLCNT, s), self.PGOPT['emllog'])
1249
- if bidx == self.ALLCNT: return
1250
- self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
1251
- chksize = self.PGLOG['DSCHECK']['size']
1252
- reorder = errcnt = self.MODCNT = bgcnt = acnt = ocnt = 0
1253
- perrcnt = self.ALLCNT
1254
- efiles = [1]*self.ALLCNT
1255
- self.params['LC'] = ['B']*self.ALLCNT
1256
- if self.PGSIG['BPROC'] > 1:
1257
- afiles = [None]*self.ALLCNT
1258
- ofiles = [None]*self.ALLCNT
1259
- sarchs = [None]*self.ALLCNT
1260
- fnames = None
1261
- while True:
1262
- for i in range(bidx, self.ALLCNT):
1263
- if self.PGSIG['BPROC'] < 2 and i > bidx and ((i-bidx)%20) == 0:
1264
- if self.PGLOG['DSCHECK']:
1265
- self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
1266
- if 'EM' in self.params:
1267
- self.PGLOG['PRGMSG'] = "{}/{} of {} saved file{} archived/processed".format(acnt, i, self.ALLCNT, s)
1268
- sfile = self.params['SF'][i]
1269
- if not (efiles[i] and sfile): continue
1270
- efiles[i] = 0
1271
- if 'ST' in self.params and self.params['ST'][i]:
1272
- type = self.params['ST'][i]
1273
- if self.PGOPT['SDTYP'].find(type) < 0:
1274
- self.pglog("{}-{}: Invalid Saved file Type '{}' to Archive".format(dsid, sfile, type), self.PGOPT['emerol'])
1275
- continue
1276
- else:
1277
- self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['errlog'])
1278
- continue
1279
- sfile = self.get_saved_path(i, sfile, 0, type)
1280
- afile = self.get_saved_path(i, sfile, 1, type)
1281
- ofile = self.join_paths(dsid, sfile)
1282
- sinfo = "{}-{}-{}".format(dsid, type, sfile)
1283
- pgrec = self.pgget(tname, "*", "{} and sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
1284
- if not pgrec:
1285
- self.pglog("{}: Fail to Cross {} for Saved file not in RDADB".format(sinfo, aname), self.PGOPT['emlerr'])
1286
- continue
1287
- elif pgrec['locflag'] == 'C':
1288
- self.pglog("{}: Fail to Cross {} Saved File for CGD data".format(sinfo, aname), self.PGOPT['extlog'])
1289
- if pgrec and self.params['SF'][i] != sfile: self.params['SF'][i] = sfile
1290
- sarch = oarch = 1
1291
- self.pglog(sinfo + ": Cross {} Saved file ...".format(aname), self.WARNLG)
1292
- info = self.check_local_file(afile, 0, self.PGOPT['emerol']|self.PFSIZE)
1293
- if info:
1294
- sarch = 0
1295
- elif info is not None:
1296
- errcnt += 1
1297
- efiles[i] = 1
1298
- dflags['G'] = self.PGLOG['DECSHOME']
1299
- continue
1300
- info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
1301
- if info:
1302
- oarch = 0
1303
- elif info is not None:
1304
- errcnt += 1
1305
- efiles[i] = 1
1306
- dflags['O'] = bucket
1307
- continue
1308
- if sarch and oarch:
1309
- self.pglog(sinfo + ": Cannot Cross {}, Neither Saved Nor Object file Exists".format(aname), self.PGOPT['errlog'])
1310
- continue
1311
- elif not (sarch or oarch) and pgrec['locflag'] == 'B':
1312
- self.pglog(sinfo + ": No need Cross {}, Both Saved & Object Exist".format(aname), self.PGOPT['wrnlog'])
1313
- continue
1314
- if sarch:
1315
- if not self.object_copy_local(afile, ofile, bucket, self.PGOPT['emerol']|self.OVERRIDE):
1316
- errcnt += 1
1317
- efiles[i] = 1
1318
- dflags['G'] = self.PGLOG['DECSHOME']
1319
- continue
1320
- if aname == 'Move':
1321
- self.params['LC'][i] = 'G'
1322
- self.delete_object_file(ofile, bucket, self.PGOPT['extlog'])
1323
- acnt += 1
1324
- elif oarch:
1325
- if not self.local_copy_object(ofile, afile, bucket, None, self.PGOPT['emerol']|self.OVERRIDE):
1326
- errcnt += 1
1327
- efiles[i] = 1
1328
- dflags['O'] = bucket
1329
- continue
1330
- if aname == 'Move':
1331
- self.params['LC'][i] = 'O'
1332
- self.delete_local_file(afile, self.PGOPT['extlog'])
1333
- ocnt += 1
1334
- if self.PGSIG['BPROC'] > 1:
1335
- afiles[i] = afile
1336
- ofiles[i] = ofile
1337
- sarchs[i] = sarch
1338
- bgcnt += 1
1339
- if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
1340
- if self.PGSIG['BPROC'] < 2:
1341
- if not fnames: fnames = self.get_field_keys(tname, None, "G") # get setting fields if not yet
1342
- info = self.get_file_origin_info(sfile, pgrec)
1343
- sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
1344
- if not sid:
1345
- self.params['SF'][i] = None
1346
- continue
1347
- if errcnt == 0 or errcnt >= perrcnt or self.PGLOG['DSCHECK']: break
1348
- perrcnt = errcnt
1349
- self.pglog("Recopy {} Saved file{} for {}".format(errcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
1350
- errcnt = self.reset_errmsg(0)
1351
- if errcnt:
1352
- self.RETSTAT = self.reset_errmsg(errcnt)
1353
- for i in range(bidx, self.ALLCNT):
1354
- if efiles[i]: self.params['SF'][i] = ''
1355
- if self.PGLOG['DSCHECK']:
1356
- self.check_storage_dflags(dflags, self.PGLOG['DSCHECK'], self.PGOPT['emerol'])
1357
- if bgcnt:
1358
- self.check_background(None, 0, self.LOGWRN, 1)
1359
- for i in range(self.ALLCNT):
1360
- if sarchs[i]:
1361
- self.validate_gladearch(afiles[i], "{}-{}".format(bucket, ofiles[i]), i)
1362
- elif ofiles[i]:
1363
- self.validate_objectarch(ofiles[i], afiles[i], bucket, i)
1364
- astr = 'Moved' if aname == 'Move' else 'Copied'
1365
- if acnt > 0: self.pglog("{} of {} Saved file{} Cross {} for {}".format(acnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
1366
- if ocnt > 0: self.pglog("{} of {} Object file{} Cross {} for {}".format(ocnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
1367
- if self.PGLOG['DSCHECK']:
1368
- self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
1369
- self.pglog("{} of {} Saved file record{} modified for {}!".format(self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
1370
- if 'ON' in self.params: reorder = self.reorder_files(dsid, self.params['ON'], tname)
1371
- if (self.MODCNT + reorder) > 0:
1372
- self.reset_rdadb_version(dsid)
1373
- if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
1374
-
1375
1178
  # get backup file names in RDADB and on Quaser server for given dataset id
1376
1179
  def get_backup_filenames(self, bfile, dsid):
1377
1180
  ms = re.match(r'^/{}/(.+)$'.format(dsid), bfile)
@@ -1422,7 +1225,7 @@ class DsArch(PgArch, PgMeta):
1422
1225
  ifidx += 1
1423
1226
  ccnt *= 3
1424
1227
  self.set_dscheck_fcount(ccnt, self.PGOPT['extlog'])
1425
- tinfo = {'bid' : 0, 'size' : 0, 'cnt' : 0, 'afmt' : '', 'dfmt' : '', 'sids' : [], 'wids' : []}
1228
+ tinfo = {'bid': 0, 'size': 0, 'cnt': 0, 'afmt': '', 'dfmt': '', 'sids': [], 'wids': []}
1426
1229
  if pgbck: tinfo['bid'] = pgbck['bid']
1427
1230
  ifidx = 1
1428
1231
  while True:
@@ -1435,13 +1238,11 @@ class DsArch(PgArch, PgMeta):
1435
1238
  ifidx += 1
1436
1239
  info = self.check_local_file(tarfile, 33) # 1+32
1437
1240
  fsize = info['data_size'] if info else 0
1438
- # if fsize < tinfo['size']:
1439
- # self.pglog("{}: Backup file size {} is less than total file size {}".format(tarfile, fsize, tinfo['size']), self.PGOPT['extlog'])
1440
1241
  if fsize < self.PGLOG['ONEGBS']:
1441
1242
  self.pglog("{}: Backup file size {} is less than one GB".format(tarfile, fsize), self.PGOPT['extlog'])
1442
- record = {'type' : qtype, 'data_format' : tinfo['dfmt'], 'data_size' : fsize,
1443
- 'uid' : self.PGOPT['UID'], 'checksum' : info['checksum'],
1444
- 'scount' : scnt, 'wcount' : wcnt}
1243
+ record = {'type': qtype, 'data_format': tinfo['dfmt'], 'data_size': fsize,
1244
+ 'uid': self.PGOPT['UID'], 'checksum': info['checksum'],
1245
+ 'scount': scnt, 'wcount': wcnt}
1445
1246
  record['file_format'] = self.append_format_string(tinfo['afmt'], tfmt, 1)
1446
1247
  record['date_created'] = record['date_modified'] = info['date_modified']
1447
1248
  record['time_created'] = record['time_modified'] = info['time_modified']
@@ -1470,7 +1271,7 @@ class DsArch(PgArch, PgMeta):
1470
1271
  self.pglog("{}: Error add Quaser Backup file name in RDADB for {}".format(bfile, dsid), self.PGOPT['extlog'])
1471
1272
  tcnt = tinfo['cnt']
1472
1273
  tsize = tinfo['size']
1473
- brec = {'bid' : bid}
1274
+ brec = {'bid': bid}
1474
1275
  if scnt:
1475
1276
  for sid in tinfo['sids']:
1476
1277
  tcnt += self.pgupdt("sfile", brec, "sid = {}".format(sid))
@@ -2240,7 +2041,7 @@ class DsArch(PgArch, PgMeta):
2240
2041
  if not pgrec or level != pgrec['level']: record['level'] = level
2241
2042
  if pgrec:
2242
2043
  modcnt += self.pgupdt(tname, record, "gidx = {}".format(pgrec['gidx']), self.LGEREX)
2243
- if pindex: tcnt += self.reset_top_group_index(dsid, gindex, 6)
2044
+ if pindex: tcnt += self.reset_top_gindex(dsid, gindex, 6)
2244
2045
  if 'grptype' in record:
2245
2046
  self.params['WN'] = 6
2246
2047
  self.CHGGRPS[gindex] = 1
@@ -2262,7 +2063,7 @@ class DsArch(PgArch, PgMeta):
2262
2063
  cnd = "{} AND {}".format(dcnd, gcnd)
2263
2064
  pgrecs = self.pgmget_wfile(dsid, 'wfile, type', gcnd, self.PGOPT['extlog'])
2264
2065
  cnt = len(pgrecs['wfile']) if pgrecs else 0
2265
- srec = {'status' : 'I'}
2066
+ srec = {'status': 'I'}
2266
2067
  if cnt:
2267
2068
  s = 's' if cnt > 1 else ''
2268
2069
  self.pglog("{}: set {} web file{} to Internal".format(grp, cnt, s), self.PGOPT['wrnlog'])
@@ -3290,7 +3091,6 @@ class DsArch(PgArch, PgMeta):
3290
3091
  tname = 'wfile'
3291
3092
  dsid = self.params['DS']
3292
3093
  dcnd = "dsid = '{}'".format(dsid)
3293
- frombucket = "gdex-decsdata"
3294
3094
  tobucket = self.PGLOG['OBJCTBKT']
3295
3095
  s = 's' if self.ALLCNT > 1 else ''
3296
3096
  bidx = chksize = 0
@@ -3319,13 +3119,11 @@ class DsArch(PgArch, PgMeta):
3319
3119
  if tmpgs: self.params['GI'] = self.params['OG']
3320
3120
  aolds = [None]*self.ALLCNT
3321
3121
  solds = [None]*self.ALLCNT
3322
- oolds = [None]*self.ALLCNT
3323
3122
  tolds = [None]*self.ALLCNT
3324
3123
  for i in range(self.ALLCNT):
3325
3124
  type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'V'
3326
3125
  aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
3327
3126
  solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
3328
- oolds[i] = self.join_paths(self.params['DS'], solds[i])
3329
3127
  tolds[i] = type
3330
3128
  if tmpds: self.params['DS'] = tmpds
3331
3129
  if tmpgs: self.params['GI'] = tmpgs
@@ -3392,14 +3190,8 @@ class DsArch(PgArch, PgMeta):
3392
3190
  not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
3393
3191
  self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
3394
3192
  "Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
3395
- sfrom = omove = wmove = 1
3396
- ofrom = 0
3193
+ omove = wmove = sfrom = 1
3397
3194
  locflag = 'G'
3398
- # locflag = pgrec['locflag']
3399
- # if locflag == 'O':
3400
- # sfrom = 0
3401
- # elif locflag == 'G':
3402
- # ofrom = 0
3403
3195
  if not self.params['LC'][i] or self.params['LC'][i] == 'R':
3404
3196
  self.params['LC'][i] = locflag
3405
3197
  else:
@@ -3411,31 +3203,21 @@ class DsArch(PgArch, PgMeta):
3411
3203
  omove = 0
3412
3204
  dslocflags.add('G')
3413
3205
  if wmove:
3414
- if sfrom:
3415
- stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
3416
- sfrom = 0
3417
- else:
3418
- stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
3206
+ stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
3207
+ sfrom = 0
3419
3208
  if not stat:
3420
3209
  self.RETSTAT = 1
3421
3210
  continue
3422
3211
  wcnt += 1
3423
3212
  if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
3424
3213
  if omove:
3425
- if sfrom:
3426
- stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
3427
- elif wmove:
3428
- stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
3429
- else:
3430
- stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
3431
- ofrom = 0
3214
+ stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
3432
3215
  if not stat:
3433
3216
  self.RETSTAT = 1
3434
3217
  continue
3435
3218
  ocnt += 1
3436
3219
  if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
3437
3220
  if sfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
3438
- if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
3439
3221
  if pgrec['bid'] and not self.params['QF'][i]: self.params['QF'][i] = pgrec['bid']
3440
3222
  if pgrec['gindex'] and not self.params['GI'][i]: self.params['GI'][i] = pgrec['gindex']
3441
3223
  if pgrec['data_size'] and not self.params['SZ'][i]: self.params['SZ'][i] = pgrec['data_size']
@@ -3549,7 +3331,7 @@ class DsArch(PgArch, PgMeta):
3549
3331
  wcnd = "wid = {}".format(pgrec['wid'])
3550
3332
  if (oflag + wflag) > 0:
3551
3333
  locflag = "O" if oflag else "G"
3552
- lrec = {'locflag' : locflag}
3334
+ lrec = {'locflag': locflag}
3553
3335
  mcnt += self.pgupdt_wfile(dsid, lrec, wcnd, self.LGEREX)
3554
3336
  else:
3555
3337
  ccnt = self.record_filenumber(dsid, pgrec['gindex'], 4, (pgrec['type'] if pgrec['status'] == 'P' else ''), -1, -pgrec['data_size'])
@@ -3683,13 +3465,11 @@ class DsArch(PgArch, PgMeta):
3683
3465
  if tmpgs: self.params['GI'] = self.params['OG']
3684
3466
  aolds = [None]*self.ALLCNT
3685
3467
  solds = [None]*self.ALLCNT
3686
- oolds = [None]*self.ALLCNT
3687
3468
  tolds = [None]*self.ALLCNT
3688
3469
  for i in range(self.ALLCNT):
3689
3470
  type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'P'
3690
3471
  aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
3691
3472
  solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
3692
- oolds[i] = self.get_object_path(solds[i], self.params['DS'])
3693
3473
  tolds[i] = type
3694
3474
  if tmpds: self.params['DS'] = tmpds
3695
3475
  if tmpgs: self.params['GI'] = tmpgs
@@ -3697,13 +3477,11 @@ class DsArch(PgArch, PgMeta):
3697
3477
  init = 1 if (tmpds or tmpgs) else 0
3698
3478
  anews = [None]*self.ALLCNT
3699
3479
  snews = [None]*self.ALLCNT
3700
- onews = [None]*self.ALLCNT
3701
3480
  tnews = [None]*self.ALLCNT
3702
3481
  for i in range(self.ALLCNT):
3703
3482
  type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'P'
3704
3483
  anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
3705
3484
  snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
3706
- onews[i] = self.get_object_path(snews[i], dsid)
3707
3485
  tnews[i] = type
3708
3486
  init = 0
3709
3487
  fnames = "FIT"
@@ -3715,13 +3493,13 @@ class DsArch(PgArch, PgMeta):
3715
3493
  if bidx == self.ALLCNT: return
3716
3494
  self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
3717
3495
  chksize = self.PGLOG['DSCHECK']['size']
3718
- reorder = self.MODCNT = scnt = ocnt = 0
3496
+ reorder = self.MODCNT = scnt = 0
3719
3497
  for i in range(bidx, self.ALLCNT):
3720
3498
  if i > bidx and ((i-bidx)%20) == 0:
3721
3499
  if self.PGLOG['DSCHECK']:
3722
3500
  self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
3723
3501
  if 'EM' in self.params:
3724
- self.PGLOG['PRGMSG'] = "{}/{}/{}/{}, Disk/Object/Record/Proccessed, of {}self.ALLCNT Saved files moved".format(scnt, ocnt, self.MODCNT, i, self.ALLCNT, s)
3502
+ self.PGLOG['PRGMSG'] = "{}/{}/{}, Disk/Record/Proccessed, of {} Saved file{} moved".format(scnt, self.MODCNT, i, self.ALLCNT, s)
3725
3503
  type = tolds[i]
3726
3504
  pgrec = self.pgget(tname, "*", "sfile = '{}' AND dsid = '{}' AND type = '{}'".format(solds[i], self.params['OD'], type), self.LGEREX)
3727
3505
  if not pgrec:
@@ -3746,27 +3524,15 @@ class DsArch(PgArch, PgMeta):
3746
3524
  not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
3747
3525
  self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
3748
3526
  "Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
3749
- omove = 0
3750
- smove = 1
3751
- # if pgrec['locflag'] == 'O':
3752
- # smove = 0
3753
- # elif pgrec['locflag'] == 'G':
3754
- # omove = 0
3755
- if smove and aolds[i] != anews[i]:
3527
+ if aolds[i] != anews[i]:
3756
3528
  if not self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE):
3757
3529
  self.RETSTAT = 1
3758
3530
  continue
3759
3531
  if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
3760
3532
  scnt += 1
3761
- if omove and oolds[i] != onews[i]:
3762
- if not self.move_object_file(onews[i], oolds[i], bucket, bucket, self.PGOPT['emerol']|self.OVERRIDE):
3763
- self.RETSTAT = 1
3764
- continue
3765
- if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
3766
- ocnt += 1
3767
3533
  self.set_one_savedfile(i, pgrec, snews[i], fnames, tnews[i], None, tmpds)
3768
3534
  if pgrec['bid']: self.save_move_info(pgrec['bid'], solds[i], type, 'S', self.params['OD'], snews[i], tnews[i], 'S', dsid)
3769
- self.pglog("{}/{}/{}, Disk/Object/Record, of {} Saved file{} moved".format(scnt, ocnt, self.MODCNT, self.ALLCNT, s), self.PGOPT['emllog'])
3535
+ self.pglog("{}/{}, Disk/Record, of {} Saved file{} moved".format(scnt, self.MODCNT, self.ALLCNT, s), self.PGOPT['emllog'])
3770
3536
  if self.PGLOG['DSCHECK']:
3771
3537
  self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
3772
3538
  if 'ON' in self.params:
@@ -3881,7 +3647,6 @@ class DsArch(PgArch, PgMeta):
3881
3647
  dsid = self.params['DS']
3882
3648
  dcnd = "dsid = '{}'".format(dsid)
3883
3649
  frombucket = self.PGLOG['OBJCTBKT']
3884
- tobucket = "gdex-decsdata"
3885
3650
  bidx = chksize = 0
3886
3651
  tmpds = tmpgs = None
3887
3652
  if 'OD' not in self.params:
@@ -3922,20 +3687,18 @@ class DsArch(PgArch, PgMeta):
3922
3687
  init = 1 if (tmpds or tmpgs) else 0
3923
3688
  anews = [None]*self.ALLCNT
3924
3689
  snews = [None]*self.ALLCNT
3925
- onews = [None]*self.ALLCNT
3926
3690
  tnews = [None]*self.ALLCNT
3927
3691
  for i in range(self.ALLCNT):
3928
3692
  type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'V'
3929
3693
  anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
3930
3694
  snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
3931
- onews[i] = self.join_paths(dsid, snews[i])
3932
3695
  tnews[i] = type
3933
3696
  init = 0
3934
3697
  self.validate_multiple_values(tname, self.ALLCNT)
3935
3698
  if self.PGLOG['DSCHECK']:
3936
3699
  bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
3937
3700
  if bidx > 0:
3938
- self.pglog("bidx of self.ALLCNT Web to Saved files processed for move", self.PGOPT['emllog'])
3701
+ self.pglog(f"{bidx} of {self.ALLCNT} Web to Saved files processed for move", self.PGOPT['emllog'])
3939
3702
  if bidx == self.ALLCNT: return
3940
3703
  self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
3941
3704
  chksize = self.PGLOG['DSCHECK']['size']
@@ -3962,7 +3725,7 @@ class DsArch(PgArch, PgMeta):
3962
3725
  if self.PGLOG['DSCHECK']:
3963
3726
  self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
3964
3727
  if 'EM' in self.params:
3965
- self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {}self.ALLCNT Web file{} moved".format(wcnt, ocnt, dcnt, self.ADDCNT, i, self.ALLCNT, s)
3728
+ self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {} Web file{} moved".format(wcnt, ocnt, dcnt, self.ADDCNT, i, self.ALLCNT, s)
3966
3729
  type = tolds[i]
3967
3730
  pgrec = self.pgget_wfile(self.params['OD'], "*", "wfile = '{}' AND type = '{}'".format(wolds[i], type), self.LGEREX)
3968
3731
  if not pgrec:
@@ -3981,45 +3744,23 @@ class DsArch(PgArch, PgMeta):
3981
3744
  not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
3982
3745
  self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
3983
3746
  "Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
3984
- ofrom = wfrom = smove = 1
3985
- omove = 0
3747
+ ofrom = wfrom = 1
3986
3748
  locflag = pgrec['locflag']
3987
3749
  if locflag == 'O':
3988
3750
  wfrom = 0
3989
3751
  elif locflag == 'G':
3990
3752
  ofrom = 0
3991
- # if not self.params['LC'][i] or self.params['LC'][i] == 'R':
3992
- # self.params['LC'][i] = locflag
3993
- # else:
3994
- # locflag = self.params['LC'][i]
3995
- # if locflag == 'O':
3996
- # smove = 0
3997
- # elif locflag == 'G':
3998
- # omove = 0
3999
- if smove:
4000
- if wfrom:
4001
- stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
4002
- wfrom = 0
4003
- else:
4004
- stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
4005
- if not stat:
4006
- self.RETSTAT = 1
4007
- continue
3753
+ if wfrom:
3754
+ stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
3755
+ wfrom = 0
4008
3756
  wcnt += 1
4009
- if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
4010
- if omove:
4011
- if wfrom:
4012
- stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
4013
- elif smove:
4014
- stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
4015
- else:
4016
- stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
4017
- ofrom = 0
4018
- if not stat:
4019
- self.RETSTAT = 1
4020
- continue
3757
+ else:
3758
+ stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
4021
3759
  ocnt += 1
4022
- if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
3760
+ if not stat:
3761
+ self.RETSTAT = 1
3762
+ continue
3763
+ if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
4023
3764
  if wfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
4024
3765
  if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
4025
3766
  if self.PGOPT['GXTYP'].find(type) > -1 and ('DX' in self.params or pgrec and pgrec['meta_link'] and pgrec['meta_link'] != 'N' and 'KM' not in self.params):
@@ -4053,7 +3794,7 @@ class DsArch(PgArch, PgMeta):
4053
3794
 
4054
3795
  # get date/time/size info from given file record
4055
3796
  def get_file_origin_info(self, fname, pgrec):
4056
- info = {'isfile' : (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size' : pgrec['data_size']}
3797
+ info = {'isfile': (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size': pgrec['data_size']}
4057
3798
  info['fname'] = op.basename(fname)
4058
3799
  info['date_modified'] = pgrec['date_modified']
4059
3800
  info['time_modified'] = pgrec['time_modified']
@@ -4066,10 +3807,9 @@ class DsArch(PgArch, PgMeta):
4066
3807
  tname = 'sfile'
4067
3808
  dsid = self.params['DS']
4068
3809
  dcnd = "dsid = '{}'".format(dsid)
4069
- bucket = "gdex-decsdata"
4070
3810
  s = 's' if self.ALLCNT > 1 else ''
4071
3811
  self.pglog("Delete {} Saved file{} from {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
4072
- bidx = chksize = reorder = scnt = ocnt = dcnt = 0
3812
+ bidx = chksize = reorder = scnt = dcnt = 0
4073
3813
  self.cache_group_info(self.ALLCNT, 0)
4074
3814
  self.validate_multiple_options(self.ALLCNT, ["ST", 'VI', 'QF', 'LC'])
4075
3815
  if self.PGLOG['DSCHECK']:
@@ -4084,7 +3824,7 @@ class DsArch(PgArch, PgMeta):
4084
3824
  if self.PGLOG['DSCHECK']:
4085
3825
  self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
4086
3826
  if 'EM' in self.params:
4087
- self.PGLOG['PRGMSG'] = "{}/{}/{}/i, Disk/Object/Record/processed, of {} Saved file{} deleted".format(scnt, ocnt, dcnt, i, self.ALLCNT, s)
3827
+ self.PGLOG['PRGMSG'] = "{}/{}/i, Disk/Record/processed, of {} Saved file{} deleted".format(scnt, dcnt, i, self.ALLCNT, s)
4088
3828
  sfile = self.params['SF'][i]
4089
3829
  if 'ST' in self.params and self.params['ST'][i]:
4090
3830
  type = self.params['ST'][i]
@@ -4096,24 +3836,12 @@ class DsArch(PgArch, PgMeta):
4096
3836
  if not pgrec:
4097
3837
  self.pglog("{}-{}: Type '{}' Saved file is not in RDADB".format(dsid, sfile, type), self.PGOPT['errlog'])
4098
3838
  continue
4099
- sdel = oflag = sflag = 1
4100
- odel = 0
4101
- locflag = 'G'
4102
- # locflag = pgrec['locflag']
4103
- # if locflag == 'O':
4104
- # sflag = 0
4105
- # elif locflag == 'G':
4106
- # oflag = 0
4107
- # elif locflag == 'C':
4108
- # sflag = oflag = 0
3839
+ sdel = sflag = 1
3840
+ locflag = pgrec['locflag']
3841
+ if locflag != 'G': sflag = 0
4109
3842
  if 'LC' in self.params and self.params['LC'][i]: locflag = self.params['LC'][i]
4110
- if locflag == 'O':
4111
- sdel = 0
4112
- elif locflag == 'G':
4113
- odel = 0
4114
- elif locflag == 'C':
4115
- sdel = odel = 0
4116
- if (sflag+oflag) == (sdel+odel):
3843
+ if locflag != 'G': sdel = 0
3844
+ if sflag == sdel:
4117
3845
  vindex = self.params['VI'][i] if 'VI' in self.params else pgrec['vindex']
4118
3846
  if vindex:
4119
3847
  self.pglog(sfile + ": Saved file is version controlled, add option -vi 0 to force delete", self.PGOPT['errlog'])
@@ -4130,24 +3858,15 @@ class DsArch(PgArch, PgMeta):
4130
3858
  if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
4131
3859
  elif self.check_local_file(afile) is None:
4132
3860
  sflag = 0
4133
- if odel:
4134
- ofile = self.join_paths(dsid, sfile)
4135
- if self.delete_object_file(ofile, bucket, self.PGOPT['emerol']):
4136
- ocnt += 1
4137
- oflag = 0
4138
- if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
4139
- elif self.check_object_file(ofile, bucket) is None:
4140
- oflag = 0
4141
- if (oflag + sflag) > 0:
4142
- locflag = "O" if oflag else "G"
4143
- self.pgexec("UPDATE sfile SET locflag = '{}' WHERE sid = {}".format(locflag, pgrec['sid']), self.LGEREX)
3861
+ if sflag > 0:
3862
+ self.pgexec("UPDATE sfile SET locflag = 'G' WHERE sid = {}".format(pgrec['sid']), self.LGEREX)
4144
3863
  else:
4145
3864
  ccnt = self.record_filenumber(dsid, pgrec['gindex'], 8, 'P', -1, -pgrec['data_size'])
4146
3865
  fcnt = self.pgdel_sfile("sid = {}".format(pgrec['sid']), self.LGEREX)
4147
3866
  if fcnt: dcnt += fcnt
4148
3867
  if ccnt: self.save_filenumber(dsid, 8, 1, fcnt)
4149
- if (scnt + ocnt + dcnt) > 0:
4150
- self.pglog("{}/{}/{}, Disk/Object/Record, of {} Saved file{} deleted for {}".format(scnt, ocnt, dcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
3868
+ if (scnt + dcnt) > 0:
3869
+ self.pglog("{}/{}, Disk/Record, of {} Saved file{} deleted for {}".format(scnt, dcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
4151
3870
  if self.PGLOG['DSCHECK']:
4152
3871
  self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
4153
3872
  if 'ON' in self.params:
@@ -4161,7 +3880,7 @@ class DsArch(PgArch, PgMeta):
4161
3880
  tname = 'bfile'
4162
3881
  dsid = self.params['DS']
4163
3882
  dcnd = "dsid = '{}'".format(dsid)
4164
- brec = {'bid' : 0}
3883
+ brec = {'bid': 0}
4165
3884
  bkend = "gdex-quasar"
4166
3885
  drend = "gdex-quasar-drdata"
4167
3886
  s = 's' if self.ALLCNT > 1 else ''
@@ -4244,7 +3963,7 @@ class DsArch(PgArch, PgMeta):
4244
3963
  metacnt = prdcnt = savedcnt = webcnt = rccnt = pgcnt = modcnt = 0
4245
3964
  twcnt = tscnt = 0
4246
3965
  for i in range(self.ALLCNT):
4247
- record = {'gindex' : self.params['GI'][i]}
3966
+ record = {'gindex': self.params['GI'][i]}
4248
3967
  gcnd = "gindex = {}".format(self.params['OG'][i])
4249
3968
  cnd = "{} AND {}".format(dcnd, gcnd)
4250
3969
  modcnt += self.pgupdt(tname, record, cnd, self.LGEREX)
@@ -4257,12 +3976,12 @@ class DsArch(PgArch, PgMeta):
4257
3976
  savedcnt += self.pgupdt("sfile", record, cnd, self.LGEREX)
4258
3977
  rccnt += self.pgupdt("rcrqst", record, cnd, self.LGEREX)
4259
3978
  if tgroups[i]:
4260
- tgrec = {'tindex' : self.params['GI'][i]}
3979
+ tgrec = {'tindex': self.params['GI'][i]}
4261
3980
  tcnd = "tindex = {}".format(self.params['OG'][i])
4262
3981
  cnd = "{} AND {}".format(dcnd, tcnd)
4263
3982
  twcnt += self.pgupdt_wfile(dsid, tgrec, tcnd, self.LGEREX)
4264
3983
  tscnt += self.pgupdt("sfile", tgrec, cnd, self.LGEREX)
4265
- pgrec = {'pindex' : self.params['GI'][i]}
3984
+ pgrec = {'pindex': self.params['GI'][i]}
4266
3985
  cnd = "pindex = {} AND {}".format(self.params['OG'][i], dcnd)
4267
3986
  pgcnt += self.pgupdt(tname, pgrec, cnd, self.LGEREX)
4268
3987
  self.pglog("{} of {} group{} changed".format(modcnt, self.ALLCNT, s), self.LOGWRN)
@@ -4296,7 +4015,7 @@ class DsArch(PgArch, PgMeta):
4296
4015
  cnt = 0
4297
4016
  getkeys = 1
4298
4017
  if cnt > 0:
4299
- values = {'okeys' : [], 'value' : []}
4018
+ values = {'okeys': [], 'value': []}
4300
4019
  for i in range(cnt):
4301
4020
  pgrec = self.pgget("dsokeys", "value", "{}AND okey = '{}'".fomrat(cond, kvalues[i]), self.LGWNEX)
4302
4021
  if pgrec:
@@ -4343,7 +4062,7 @@ class DsArch(PgArch, PgMeta):
4343
4062
  pgrec['value'] = value
4344
4063
  mcnt += self.pgupdt("dsokeys", pgrec, cond, self.LGWNEX)
4345
4064
  else:
4346
- pgrec = {'dsid' : dsid, 'okey' : key, 'value' : value}
4065
+ pgrec = {'dsid': dsid, 'okey': key, 'value': value}
4347
4066
  acnt += self.pgadd("dsokeys", pgrec, self.LGWNEX)
4348
4067
  self.pglog("{}/{}/{} of {} key/value pairs added/modified/deleted for {}!".format(acnt, mcnt, dcnt, cnt, dsid), self.LOGWRN)
4349
4068
  return (acnt + mcnt + dcnt)
@@ -4353,8 +4072,8 @@ class DsArch(PgArch, PgMeta):
4353
4072
  date = self.curdate()
4354
4073
  cond = "wid = {} and date = '{}'".format(pgrec['wid'], date)
4355
4074
  if not self.pgget("wmove", "", cond, self.LGWNEX):
4356
- record = {'dsid' : pgrec['dsid'], 'uid' : self.PGOPT['UID'],
4357
- 'wfile' : pgrec['wfile'], 'wid' : pgrec['wid'], 'date' : date}
4075
+ record = {'dsid': pgrec['dsid'], 'uid': self.PGOPT['UID'],
4076
+ 'wfile': pgrec['wfile'], 'wid': pgrec['wid'], 'date': date}
4358
4077
  self.pgadd("wmove", record, self.LGWNEX)
4359
4078
 
4360
4079
  # reset file counts for saved groups
@@ -4375,10 +4094,10 @@ class DsArch(PgArch, PgMeta):
4375
4094
  if 'GI' in self.params:
4376
4095
  for gindex in self.params['GI']:
4377
4096
  if gindex is None or gindex in cgidxs: continue
4378
- tcnt += self.reset_top_group_index(dsid, gindex, act)
4097
+ tcnt += self.reset_top_gindex(dsid, gindex, act)
4379
4098
  cgidxs[gindex] = gindex
4380
4099
  else:
4381
- tcnt += self.reset_top_group_index(dsid, 0, act)
4100
+ tcnt += self.reset_top_gindex(dsid, 0, act)
4382
4101
  return tcnt
4383
4102
 
4384
4103
  # set the re-archived file counts for groups
@@ -4589,7 +4308,7 @@ class DsArch(PgArch, PgMeta):
4589
4308
  addcnt += 1
4590
4309
  if not doi: vidx = 0
4591
4310
  if vidx:
4592
- vrec = {'vindex' : vidx}
4311
+ vrec = {'vindex': vidx}
4593
4312
  vcnd = "type = 'D' AND vindex = 0"
4594
4313
  fcnt = self.pgupdt_wfile(dsid, vrec, vcnd, self.PGOPT['extlog'])
4595
4314
  if fcnt > 0:
@@ -4641,7 +4360,7 @@ class DsArch(PgArch, PgMeta):
4641
4360
  self.pglog("{}: Cannot terminate for DOI {} is asscoated to Version control {}".format(vinfo, pgrec['doi'], orec['vindex']), self.LOGERR)
4642
4361
  continue
4643
4362
  doicnt += 1
4644
- record = {'status' : "H"}
4363
+ record = {'status': "H"}
4645
4364
  record['end_date'] = self.params['ED'][i] if 'ED' in self.params else self.curdate()
4646
4365
  record['end_time'] = self.params['ET'][i] if 'ET' in self.params else self.curtime()
4647
4366
  if self.pgupdt("dsvrsn", record, cnd, self.PGOPT['extlog']):
@@ -679,7 +679,7 @@ class PgMeta(PgCMD, PgSplit):
679
679
  pgrecs = self.pgmget("dsgroup", "gindex", pcnd, self.LGEREX)
680
680
  cnt = len(pgrecs['gindex']) if pgrecs else 0
681
681
  for i in range(cnt):
682
- tcnt += self.reset_top_gindex(dsid, pgrecs['gindex'][i])
682
+ tcnt += self.reset_top_gindex(dsid, pgrecs['gindex'][i], act)
683
683
  return tcnt
684
684
 
685
685
  # reset metalink in table wfile
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rda_python_dsarch
3
- Version: 2.0.1
3
+ Version: 2.0.3
4
4
  Summary: RDA Python package for utility program dsarch archive RDA data
5
5
  Author-email: Zaihua Ji <zji@ucar.edu>
6
6
  Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch
@@ -2,13 +2,13 @@ rda_python_dsarch/PgArch.py,sha256=d1Y87MT49tgQygLAffNff9Qs4c12KXspYyDG8liSnpQ,5
2
2
  rda_python_dsarch/PgMeta.py,sha256=FV4GytgnJK00FUlyM6iHqpNd7-XTVHO1j5tZtcLTWgY,26845
3
3
  rda_python_dsarch/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
4
4
  rda_python_dsarch/ds_arch.py,sha256=6bR0y6_rhIXyvbuYd-oZfL1YcaHhWBkojjzYz1yCrVg,233810
5
- rda_python_dsarch/dsarch.py,sha256=nIkMc5eIwucX6c6y-I4J3civb_mo3rd087gfM_-rrvE,246779
5
+ rda_python_dsarch/dsarch.py,sha256=dN86W98fU6vbp8DnOMpTxVkA-NTvmxLVlaMiHFfo15Y,232945
6
6
  rda_python_dsarch/dsarch.usg,sha256=vsxGneGCIyzqZx5mwQ8xkTmdNmuny56LgokgDkWt5xE,132034
7
7
  rda_python_dsarch/pg_arch.py,sha256=Ky_RfY8q0Uq7OjK1UC8npkW8v_eyk_aAN7bypecYP-w,64598
8
- rda_python_dsarch/pg_meta.py,sha256=Z9FrOJG0ie-faIZu6JNyW-m1zY80aOe_uNYWLwgzRUc,29543
9
- rda_python_dsarch-2.0.1.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
10
- rda_python_dsarch-2.0.1.dist-info/METADATA,sha256=glBH3RBbIiSlE6I3F1U3cdX5LKCGx752AHS7_4gIgLw,670
11
- rda_python_dsarch-2.0.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
12
- rda_python_dsarch-2.0.1.dist-info/entry_points.txt,sha256=GR7ShgBv8qixS2l4ufAu-jw3N09pbo9zRewwPHqaOxo,60
13
- rda_python_dsarch-2.0.1.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
14
- rda_python_dsarch-2.0.1.dist-info/RECORD,,
8
+ rda_python_dsarch/pg_meta.py,sha256=f_7w9UxhJk1i-QfkmDvNCPrrDYGcOkNPNImXLNZw0sA,29548
9
+ rda_python_dsarch-2.0.3.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
10
+ rda_python_dsarch-2.0.3.dist-info/METADATA,sha256=qZq3KrenH-LV_Np3E0NtnoM93SrEvQNiTE06qVy65vo,670
11
+ rda_python_dsarch-2.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
12
+ rda_python_dsarch-2.0.3.dist-info/entry_points.txt,sha256=GR7ShgBv8qixS2l4ufAu-jw3N09pbo9zRewwPHqaOxo,60
13
+ rda_python_dsarch-2.0.3.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
14
+ rda_python_dsarch-2.0.3.dist-info/RECORD,,