rda-python-dsarch 2.0.1__py3-none-any.whl → 2.0.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_dsarch/dsarch.py +63 -344
- rda_python_dsarch/pg_meta.py +1 -1
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/METADATA +1 -1
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/RECORD +8 -8
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/WHEEL +0 -0
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/entry_points.txt +0 -0
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/licenses/LICENSE +0 -0
- {rda_python_dsarch-2.0.1.dist-info → rda_python_dsarch-2.0.3.dist-info}/top_level.txt +0 -0
rda_python_dsarch/dsarch.py
CHANGED
|
@@ -77,13 +77,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
77
77
|
elif self.PGOPT['ACTS'] == self.OPTS['AS'][0]:
|
|
78
78
|
self.ALLCNT = len(self.params['SF']) if 'SF' in self.params else len(self.params['LF'])
|
|
79
79
|
self.cache_group_info(self.ALLCNT)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
elif 'XM' in self.params:
|
|
83
|
-
self.crosscopy_saved_files('Move')
|
|
84
|
-
else:
|
|
85
|
-
self.archive_saved_files()
|
|
86
|
-
if 'CL' in self.params: self.delete_local_files()
|
|
80
|
+
self.archive_saved_files()
|
|
81
|
+
if 'CL' in self.params: self.delete_local_files()
|
|
87
82
|
elif self.PGOPT['ACTS'] == self.OPTS['AW'][0]:
|
|
88
83
|
self.ALLCNT = len(self.params['WF']) if 'WF' in self.params else len(self.params['LF'])
|
|
89
84
|
if 'ML' in self.params and 'GX' in self.params: del self.params['ML']
|
|
@@ -306,7 +301,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
306
301
|
self.cmdlog(None, 0, self.LOGWRN|self.SNDEML)
|
|
307
302
|
else:
|
|
308
303
|
self.cmdlog()
|
|
309
|
-
self.pgexit(0)
|
|
310
304
|
|
|
311
305
|
# archive web/object files
|
|
312
306
|
def archive_web_files(self):
|
|
@@ -727,13 +721,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
727
721
|
# archive save files
|
|
728
722
|
def archive_saved_files(self):
|
|
729
723
|
tname = 'sfile'
|
|
730
|
-
dftloc =
|
|
724
|
+
dftloc = 'G'
|
|
731
725
|
dsid = self.params['DS']
|
|
732
726
|
dcnd = "dsid = '{}'".format(dsid)
|
|
733
727
|
s = 's' if self.ALLCNT > 1 else ''
|
|
734
728
|
bidx = chksize = 0
|
|
735
729
|
dflags = {}
|
|
736
|
-
bucket = "gdex-decsdata" # object store bucket
|
|
737
730
|
self.pglog("Archive {} Saved file{} of {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
|
|
738
731
|
if 'ZD' in self.params or 'UZ' in self.params:
|
|
739
732
|
self.compress_localfile_list(self.PGOPT['CACT'], self.ALLCNT)
|
|
@@ -753,18 +746,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
753
746
|
self.params['AF'] = [None]*self.ALLCNT
|
|
754
747
|
self.OPTS['AF'][2] |= 2
|
|
755
748
|
if 'LC' not in self.params:
|
|
756
|
-
self.params['LC'] = [
|
|
749
|
+
self.params['LC'] = [dftloc]*self.ALLCNT
|
|
757
750
|
self.OPTS['LC'][2] |= 2
|
|
758
751
|
if 'SZ' not in self.params:
|
|
759
752
|
self.params['SZ'] = [0]*self.ALLCNT
|
|
760
753
|
self.OPTS['SZ'][2] |= 2
|
|
761
754
|
if 'MC' not in self.params: self.params['MC'] = [None]*self.ALLCNT
|
|
762
|
-
reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt =
|
|
755
|
+
reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt = 0
|
|
763
756
|
perrcnt = self.ALLCNT
|
|
764
757
|
efiles = [1]*self.ALLCNT
|
|
765
758
|
if self.PGSIG['BPROC'] > 1:
|
|
766
759
|
afiles = [None]*self.ALLCNT
|
|
767
|
-
ofiles = [None]*self.ALLCNT
|
|
768
760
|
lfiles = [None]*self.ALLCNT
|
|
769
761
|
fnames = None
|
|
770
762
|
override = self.OVERRIDE
|
|
@@ -790,19 +782,9 @@ class DsArch(PgArch, PgMeta):
|
|
|
790
782
|
self.params['LF'][i] = self.params['SF'][i] = None
|
|
791
783
|
continue
|
|
792
784
|
locflag = self.params['LC'][i]
|
|
793
|
-
if not locflag or locflag == 'R':
|
|
794
|
-
if not dftloc: dftloc = self.get_dataset_locflag(dsid)
|
|
795
|
-
locflag = self.params['LC'][i] = dftloc
|
|
796
785
|
if locflag == 'C': self.pglog(lfile + ": Cannot Archive Saved File for CGD data", self.PGOPT['extlog'])
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
elif locflag == 'O':
|
|
800
|
-
oarch = 1
|
|
801
|
-
sarch = 0
|
|
802
|
-
else:
|
|
803
|
-
oarch = 0
|
|
804
|
-
sarch = 1
|
|
805
|
-
if oarch: self.pglog(lfile + ": Cannot Archive Saved File onto Boreas", self.PGOPT['extlog'])
|
|
786
|
+
locflag = self.params['LC'][i] = dftloc
|
|
787
|
+
sarch = 1
|
|
806
788
|
if not self.params['MC'][i]: self.params['MC'][i] = self.get_md5sum(lfile) # re-get MD5 Checksum
|
|
807
789
|
if not (self.params['SZ'][i] and self.params['SZ'][i] == lsize):
|
|
808
790
|
self.params['SZ'][i] = lsize
|
|
@@ -818,9 +800,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
818
800
|
else:
|
|
819
801
|
self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['extlog'])
|
|
820
802
|
continue
|
|
821
|
-
afile =
|
|
803
|
+
afile = self.get_saved_path(i, sfile, 1, type)
|
|
822
804
|
sfile = self.get_saved_path(i, sfile, 0, type)
|
|
823
|
-
ofile = (self.join_paths(dsid, sfile) if oarch else None)
|
|
824
805
|
pgrec = self.pgget(tname, "*", "{} AND sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
|
|
825
806
|
if not pgrec:
|
|
826
807
|
pgrec = self.pgget(tname, "type", "{} AND sfile = '{}'".format(dcnd, sfile), self.PGOPT['extlog'])
|
|
@@ -852,45 +833,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
852
833
|
efiles[i] = 1
|
|
853
834
|
dflags['G'] = self.PGLOG['DECSHOME']
|
|
854
835
|
continue
|
|
855
|
-
if
|
|
856
|
-
|
|
857
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
858
|
-
if info:
|
|
859
|
-
if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
|
|
860
|
-
self.pglog("Object-{}-{}: Same-Checksum ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['emllog'])
|
|
861
|
-
oarch = 0
|
|
862
|
-
elif info['data_size'] == lsize:
|
|
863
|
-
self.pglog("Object-{}-{}: Same-Size ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['wrnlog'])
|
|
864
|
-
oarch = 0
|
|
865
|
-
elif vsnctl and not override:
|
|
866
|
-
self.pglog("Object-{}-{}: Cannot rearchive version controlled file".format(bucket, ofile), self.PGOPT['extlog'])
|
|
867
|
-
self.params['SF'][i] = None
|
|
868
|
-
continue
|
|
869
|
-
else:
|
|
870
|
-
replace = 1
|
|
871
|
-
elif info is not None:
|
|
836
|
+
if sarch:
|
|
837
|
+
if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
|
|
872
838
|
errcnt += 1
|
|
873
839
|
efiles[i] = 1
|
|
874
|
-
dflags['O'] = bucket
|
|
875
840
|
continue
|
|
876
|
-
|
|
877
|
-
if sarch:
|
|
878
|
-
if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
|
|
879
|
-
errcnt += 1
|
|
880
|
-
efiles[i] = 1
|
|
881
|
-
continue
|
|
882
|
-
acnt += 1
|
|
883
|
-
if oarch:
|
|
884
|
-
if replace: self.delete_object_file(ofile, bucket)
|
|
885
|
-
if not self.local_copy_object(ofile, lfile, bucket, None, self.PGOPT['emerol']|override):
|
|
886
|
-
errcnt += 1
|
|
887
|
-
efiles[i] = 1
|
|
888
|
-
continue
|
|
889
|
-
ocnt += 1
|
|
841
|
+
acnt += 1
|
|
890
842
|
if self.PGLOG['DSCHECK']: chksize += lsize
|
|
891
843
|
if self.PGSIG['BPROC'] > 1:
|
|
892
844
|
afiles[i] = afile
|
|
893
|
-
ofiles[i] = ofile
|
|
894
845
|
lfiles[i] = lfile
|
|
895
846
|
bgcnt += 1
|
|
896
847
|
if self.PGSIG['BPROC'] < 2:
|
|
@@ -898,12 +849,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
898
849
|
info = None
|
|
899
850
|
if sarch:
|
|
900
851
|
info = self.check_local_file(afile, 1, self.PGOPT['emerol']|self.PFSIZE)
|
|
901
|
-
elif oarch:
|
|
902
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
903
852
|
elif pgrec:
|
|
904
853
|
info = self.get_file_origin_info(sfile, pgrec)
|
|
905
|
-
elif locflag == 'O':
|
|
906
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
907
854
|
sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
|
|
908
855
|
if not sid:
|
|
909
856
|
self.params['LF'][i] = self.params['SF'][i] = None
|
|
@@ -923,11 +870,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
923
870
|
self.check_background(None, 0, self.LOGWRN, 1)
|
|
924
871
|
for i in range(self.ALLCNT):
|
|
925
872
|
if afiles[i]: self.validate_gladearch(afiles[i], lfiles[i], i)
|
|
926
|
-
if ofiles[i]: self.validate_objectarch(ofiles[i], lfiles[i], bucket, i)
|
|
927
873
|
if acnt > 0:
|
|
928
874
|
self.pglog("{} of {} Saved file{} archived for {}".format(acnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
929
|
-
if ocnt > 0:
|
|
930
|
-
self.pglog("{] of {} Object file{} archived for {}".format(ocnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
931
875
|
if self.PGLOG['DSCHECK']:
|
|
932
876
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
933
877
|
self.pglog("{}/{} of {} Saved file record{} added/modified for {}!".format(self.ADDCNT, self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
@@ -1231,147 +1175,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
1231
1175
|
self.reset_rdadb_version(dsid)
|
|
1232
1176
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1233
1177
|
|
|
1234
|
-
# cross copy save files between glade and object store
|
|
1235
|
-
def crosscopy_saved_files(self, aname):
|
|
1236
|
-
tname = 'sfile'
|
|
1237
|
-
dsid = self.params['DS']
|
|
1238
|
-
dcnd = "dsid = '{}'".format(dsid)
|
|
1239
|
-
s = 's' if self.ALLCNT > 1 else ''
|
|
1240
|
-
bidx = chksize = 0
|
|
1241
|
-
dflags = {}
|
|
1242
|
-
bucket = "gdex-decsdata"
|
|
1243
|
-
self.pglog("Cross {} {} Saved file{} of {} ...".format(aname, self.ALLCNT, s, dsid), self.WARNLG)
|
|
1244
|
-
self.validate_multiple_values(tname, self.ALLCNT)
|
|
1245
|
-
if self.PGLOG['DSCHECK']:
|
|
1246
|
-
bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
|
|
1247
|
-
if bidx > 0:
|
|
1248
|
-
self.pglog("{} of {} file{} processed for SAVED archive".format(bidx, self.ALLCNT, s), self.PGOPT['emllog'])
|
|
1249
|
-
if bidx == self.ALLCNT: return
|
|
1250
|
-
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
1251
|
-
chksize = self.PGLOG['DSCHECK']['size']
|
|
1252
|
-
reorder = errcnt = self.MODCNT = bgcnt = acnt = ocnt = 0
|
|
1253
|
-
perrcnt = self.ALLCNT
|
|
1254
|
-
efiles = [1]*self.ALLCNT
|
|
1255
|
-
self.params['LC'] = ['B']*self.ALLCNT
|
|
1256
|
-
if self.PGSIG['BPROC'] > 1:
|
|
1257
|
-
afiles = [None]*self.ALLCNT
|
|
1258
|
-
ofiles = [None]*self.ALLCNT
|
|
1259
|
-
sarchs = [None]*self.ALLCNT
|
|
1260
|
-
fnames = None
|
|
1261
|
-
while True:
|
|
1262
|
-
for i in range(bidx, self.ALLCNT):
|
|
1263
|
-
if self.PGSIG['BPROC'] < 2 and i > bidx and ((i-bidx)%20) == 0:
|
|
1264
|
-
if self.PGLOG['DSCHECK']:
|
|
1265
|
-
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
1266
|
-
if 'EM' in self.params:
|
|
1267
|
-
self.PGLOG['PRGMSG'] = "{}/{} of {} saved file{} archived/processed".format(acnt, i, self.ALLCNT, s)
|
|
1268
|
-
sfile = self.params['SF'][i]
|
|
1269
|
-
if not (efiles[i] and sfile): continue
|
|
1270
|
-
efiles[i] = 0
|
|
1271
|
-
if 'ST' in self.params and self.params['ST'][i]:
|
|
1272
|
-
type = self.params['ST'][i]
|
|
1273
|
-
if self.PGOPT['SDTYP'].find(type) < 0:
|
|
1274
|
-
self.pglog("{}-{}: Invalid Saved file Type '{}' to Archive".format(dsid, sfile, type), self.PGOPT['emerol'])
|
|
1275
|
-
continue
|
|
1276
|
-
else:
|
|
1277
|
-
self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['errlog'])
|
|
1278
|
-
continue
|
|
1279
|
-
sfile = self.get_saved_path(i, sfile, 0, type)
|
|
1280
|
-
afile = self.get_saved_path(i, sfile, 1, type)
|
|
1281
|
-
ofile = self.join_paths(dsid, sfile)
|
|
1282
|
-
sinfo = "{}-{}-{}".format(dsid, type, sfile)
|
|
1283
|
-
pgrec = self.pgget(tname, "*", "{} and sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
|
|
1284
|
-
if not pgrec:
|
|
1285
|
-
self.pglog("{}: Fail to Cross {} for Saved file not in RDADB".format(sinfo, aname), self.PGOPT['emlerr'])
|
|
1286
|
-
continue
|
|
1287
|
-
elif pgrec['locflag'] == 'C':
|
|
1288
|
-
self.pglog("{}: Fail to Cross {} Saved File for CGD data".format(sinfo, aname), self.PGOPT['extlog'])
|
|
1289
|
-
if pgrec and self.params['SF'][i] != sfile: self.params['SF'][i] = sfile
|
|
1290
|
-
sarch = oarch = 1
|
|
1291
|
-
self.pglog(sinfo + ": Cross {} Saved file ...".format(aname), self.WARNLG)
|
|
1292
|
-
info = self.check_local_file(afile, 0, self.PGOPT['emerol']|self.PFSIZE)
|
|
1293
|
-
if info:
|
|
1294
|
-
sarch = 0
|
|
1295
|
-
elif info is not None:
|
|
1296
|
-
errcnt += 1
|
|
1297
|
-
efiles[i] = 1
|
|
1298
|
-
dflags['G'] = self.PGLOG['DECSHOME']
|
|
1299
|
-
continue
|
|
1300
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
1301
|
-
if info:
|
|
1302
|
-
oarch = 0
|
|
1303
|
-
elif info is not None:
|
|
1304
|
-
errcnt += 1
|
|
1305
|
-
efiles[i] = 1
|
|
1306
|
-
dflags['O'] = bucket
|
|
1307
|
-
continue
|
|
1308
|
-
if sarch and oarch:
|
|
1309
|
-
self.pglog(sinfo + ": Cannot Cross {}, Neither Saved Nor Object file Exists".format(aname), self.PGOPT['errlog'])
|
|
1310
|
-
continue
|
|
1311
|
-
elif not (sarch or oarch) and pgrec['locflag'] == 'B':
|
|
1312
|
-
self.pglog(sinfo + ": No need Cross {}, Both Saved & Object Exist".format(aname), self.PGOPT['wrnlog'])
|
|
1313
|
-
continue
|
|
1314
|
-
if sarch:
|
|
1315
|
-
if not self.object_copy_local(afile, ofile, bucket, self.PGOPT['emerol']|self.OVERRIDE):
|
|
1316
|
-
errcnt += 1
|
|
1317
|
-
efiles[i] = 1
|
|
1318
|
-
dflags['G'] = self.PGLOG['DECSHOME']
|
|
1319
|
-
continue
|
|
1320
|
-
if aname == 'Move':
|
|
1321
|
-
self.params['LC'][i] = 'G'
|
|
1322
|
-
self.delete_object_file(ofile, bucket, self.PGOPT['extlog'])
|
|
1323
|
-
acnt += 1
|
|
1324
|
-
elif oarch:
|
|
1325
|
-
if not self.local_copy_object(ofile, afile, bucket, None, self.PGOPT['emerol']|self.OVERRIDE):
|
|
1326
|
-
errcnt += 1
|
|
1327
|
-
efiles[i] = 1
|
|
1328
|
-
dflags['O'] = bucket
|
|
1329
|
-
continue
|
|
1330
|
-
if aname == 'Move':
|
|
1331
|
-
self.params['LC'][i] = 'O'
|
|
1332
|
-
self.delete_local_file(afile, self.PGOPT['extlog'])
|
|
1333
|
-
ocnt += 1
|
|
1334
|
-
if self.PGSIG['BPROC'] > 1:
|
|
1335
|
-
afiles[i] = afile
|
|
1336
|
-
ofiles[i] = ofile
|
|
1337
|
-
sarchs[i] = sarch
|
|
1338
|
-
bgcnt += 1
|
|
1339
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
1340
|
-
if self.PGSIG['BPROC'] < 2:
|
|
1341
|
-
if not fnames: fnames = self.get_field_keys(tname, None, "G") # get setting fields if not yet
|
|
1342
|
-
info = self.get_file_origin_info(sfile, pgrec)
|
|
1343
|
-
sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
|
|
1344
|
-
if not sid:
|
|
1345
|
-
self.params['SF'][i] = None
|
|
1346
|
-
continue
|
|
1347
|
-
if errcnt == 0 or errcnt >= perrcnt or self.PGLOG['DSCHECK']: break
|
|
1348
|
-
perrcnt = errcnt
|
|
1349
|
-
self.pglog("Recopy {} Saved file{} for {}".format(errcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
1350
|
-
errcnt = self.reset_errmsg(0)
|
|
1351
|
-
if errcnt:
|
|
1352
|
-
self.RETSTAT = self.reset_errmsg(errcnt)
|
|
1353
|
-
for i in range(bidx, self.ALLCNT):
|
|
1354
|
-
if efiles[i]: self.params['SF'][i] = ''
|
|
1355
|
-
if self.PGLOG['DSCHECK']:
|
|
1356
|
-
self.check_storage_dflags(dflags, self.PGLOG['DSCHECK'], self.PGOPT['emerol'])
|
|
1357
|
-
if bgcnt:
|
|
1358
|
-
self.check_background(None, 0, self.LOGWRN, 1)
|
|
1359
|
-
for i in range(self.ALLCNT):
|
|
1360
|
-
if sarchs[i]:
|
|
1361
|
-
self.validate_gladearch(afiles[i], "{}-{}".format(bucket, ofiles[i]), i)
|
|
1362
|
-
elif ofiles[i]:
|
|
1363
|
-
self.validate_objectarch(ofiles[i], afiles[i], bucket, i)
|
|
1364
|
-
astr = 'Moved' if aname == 'Move' else 'Copied'
|
|
1365
|
-
if acnt > 0: self.pglog("{} of {} Saved file{} Cross {} for {}".format(acnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
|
|
1366
|
-
if ocnt > 0: self.pglog("{} of {} Object file{} Cross {} for {}".format(ocnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
|
|
1367
|
-
if self.PGLOG['DSCHECK']:
|
|
1368
|
-
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
1369
|
-
self.pglog("{} of {} Saved file record{} modified for {}!".format(self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
1370
|
-
if 'ON' in self.params: reorder = self.reorder_files(dsid, self.params['ON'], tname)
|
|
1371
|
-
if (self.MODCNT + reorder) > 0:
|
|
1372
|
-
self.reset_rdadb_version(dsid)
|
|
1373
|
-
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1374
|
-
|
|
1375
1178
|
# get backup file names in RDADB and on Quaser server for given dataset id
|
|
1376
1179
|
def get_backup_filenames(self, bfile, dsid):
|
|
1377
1180
|
ms = re.match(r'^/{}/(.+)$'.format(dsid), bfile)
|
|
@@ -1422,7 +1225,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
1422
1225
|
ifidx += 1
|
|
1423
1226
|
ccnt *= 3
|
|
1424
1227
|
self.set_dscheck_fcount(ccnt, self.PGOPT['extlog'])
|
|
1425
|
-
tinfo = {'bid'
|
|
1228
|
+
tinfo = {'bid': 0, 'size': 0, 'cnt': 0, 'afmt': '', 'dfmt': '', 'sids': [], 'wids': []}
|
|
1426
1229
|
if pgbck: tinfo['bid'] = pgbck['bid']
|
|
1427
1230
|
ifidx = 1
|
|
1428
1231
|
while True:
|
|
@@ -1435,13 +1238,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
1435
1238
|
ifidx += 1
|
|
1436
1239
|
info = self.check_local_file(tarfile, 33) # 1+32
|
|
1437
1240
|
fsize = info['data_size'] if info else 0
|
|
1438
|
-
# if fsize < tinfo['size']:
|
|
1439
|
-
# self.pglog("{}: Backup file size {} is less than total file size {}".format(tarfile, fsize, tinfo['size']), self.PGOPT['extlog'])
|
|
1440
1241
|
if fsize < self.PGLOG['ONEGBS']:
|
|
1441
1242
|
self.pglog("{}: Backup file size {} is less than one GB".format(tarfile, fsize), self.PGOPT['extlog'])
|
|
1442
|
-
record = {'type'
|
|
1443
|
-
'uid'
|
|
1444
|
-
'scount'
|
|
1243
|
+
record = {'type': qtype, 'data_format': tinfo['dfmt'], 'data_size': fsize,
|
|
1244
|
+
'uid': self.PGOPT['UID'], 'checksum': info['checksum'],
|
|
1245
|
+
'scount': scnt, 'wcount': wcnt}
|
|
1445
1246
|
record['file_format'] = self.append_format_string(tinfo['afmt'], tfmt, 1)
|
|
1446
1247
|
record['date_created'] = record['date_modified'] = info['date_modified']
|
|
1447
1248
|
record['time_created'] = record['time_modified'] = info['time_modified']
|
|
@@ -1470,7 +1271,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
1470
1271
|
self.pglog("{}: Error add Quaser Backup file name in RDADB for {}".format(bfile, dsid), self.PGOPT['extlog'])
|
|
1471
1272
|
tcnt = tinfo['cnt']
|
|
1472
1273
|
tsize = tinfo['size']
|
|
1473
|
-
brec = {'bid'
|
|
1274
|
+
brec = {'bid': bid}
|
|
1474
1275
|
if scnt:
|
|
1475
1276
|
for sid in tinfo['sids']:
|
|
1476
1277
|
tcnt += self.pgupdt("sfile", brec, "sid = {}".format(sid))
|
|
@@ -2240,7 +2041,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
2240
2041
|
if not pgrec or level != pgrec['level']: record['level'] = level
|
|
2241
2042
|
if pgrec:
|
|
2242
2043
|
modcnt += self.pgupdt(tname, record, "gidx = {}".format(pgrec['gidx']), self.LGEREX)
|
|
2243
|
-
if pindex: tcnt += self.
|
|
2044
|
+
if pindex: tcnt += self.reset_top_gindex(dsid, gindex, 6)
|
|
2244
2045
|
if 'grptype' in record:
|
|
2245
2046
|
self.params['WN'] = 6
|
|
2246
2047
|
self.CHGGRPS[gindex] = 1
|
|
@@ -2262,7 +2063,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
2262
2063
|
cnd = "{} AND {}".format(dcnd, gcnd)
|
|
2263
2064
|
pgrecs = self.pgmget_wfile(dsid, 'wfile, type', gcnd, self.PGOPT['extlog'])
|
|
2264
2065
|
cnt = len(pgrecs['wfile']) if pgrecs else 0
|
|
2265
|
-
srec = {'status'
|
|
2066
|
+
srec = {'status': 'I'}
|
|
2266
2067
|
if cnt:
|
|
2267
2068
|
s = 's' if cnt > 1 else ''
|
|
2268
2069
|
self.pglog("{}: set {} web file{} to Internal".format(grp, cnt, s), self.PGOPT['wrnlog'])
|
|
@@ -3290,7 +3091,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
3290
3091
|
tname = 'wfile'
|
|
3291
3092
|
dsid = self.params['DS']
|
|
3292
3093
|
dcnd = "dsid = '{}'".format(dsid)
|
|
3293
|
-
frombucket = "gdex-decsdata"
|
|
3294
3094
|
tobucket = self.PGLOG['OBJCTBKT']
|
|
3295
3095
|
s = 's' if self.ALLCNT > 1 else ''
|
|
3296
3096
|
bidx = chksize = 0
|
|
@@ -3319,13 +3119,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3319
3119
|
if tmpgs: self.params['GI'] = self.params['OG']
|
|
3320
3120
|
aolds = [None]*self.ALLCNT
|
|
3321
3121
|
solds = [None]*self.ALLCNT
|
|
3322
|
-
oolds = [None]*self.ALLCNT
|
|
3323
3122
|
tolds = [None]*self.ALLCNT
|
|
3324
3123
|
for i in range(self.ALLCNT):
|
|
3325
3124
|
type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'V'
|
|
3326
3125
|
aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
|
|
3327
3126
|
solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
|
|
3328
|
-
oolds[i] = self.join_paths(self.params['DS'], solds[i])
|
|
3329
3127
|
tolds[i] = type
|
|
3330
3128
|
if tmpds: self.params['DS'] = tmpds
|
|
3331
3129
|
if tmpgs: self.params['GI'] = tmpgs
|
|
@@ -3392,14 +3190,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
3392
3190
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3393
3191
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3394
3192
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3395
|
-
|
|
3396
|
-
ofrom = 0
|
|
3193
|
+
omove = wmove = sfrom = 1
|
|
3397
3194
|
locflag = 'G'
|
|
3398
|
-
# locflag = pgrec['locflag']
|
|
3399
|
-
# if locflag == 'O':
|
|
3400
|
-
# sfrom = 0
|
|
3401
|
-
# elif locflag == 'G':
|
|
3402
|
-
# ofrom = 0
|
|
3403
3195
|
if not self.params['LC'][i] or self.params['LC'][i] == 'R':
|
|
3404
3196
|
self.params['LC'][i] = locflag
|
|
3405
3197
|
else:
|
|
@@ -3411,31 +3203,21 @@ class DsArch(PgArch, PgMeta):
|
|
|
3411
3203
|
omove = 0
|
|
3412
3204
|
dslocflags.add('G')
|
|
3413
3205
|
if wmove:
|
|
3414
|
-
|
|
3415
|
-
|
|
3416
|
-
sfrom = 0
|
|
3417
|
-
else:
|
|
3418
|
-
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3206
|
+
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
3207
|
+
sfrom = 0
|
|
3419
3208
|
if not stat:
|
|
3420
3209
|
self.RETSTAT = 1
|
|
3421
3210
|
continue
|
|
3422
3211
|
wcnt += 1
|
|
3423
3212
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3424
3213
|
if omove:
|
|
3425
|
-
|
|
3426
|
-
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3427
|
-
elif wmove:
|
|
3428
|
-
stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3429
|
-
else:
|
|
3430
|
-
stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3431
|
-
ofrom = 0
|
|
3214
|
+
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3432
3215
|
if not stat:
|
|
3433
3216
|
self.RETSTAT = 1
|
|
3434
3217
|
continue
|
|
3435
3218
|
ocnt += 1
|
|
3436
3219
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3437
3220
|
if sfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
|
|
3438
|
-
if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
|
|
3439
3221
|
if pgrec['bid'] and not self.params['QF'][i]: self.params['QF'][i] = pgrec['bid']
|
|
3440
3222
|
if pgrec['gindex'] and not self.params['GI'][i]: self.params['GI'][i] = pgrec['gindex']
|
|
3441
3223
|
if pgrec['data_size'] and not self.params['SZ'][i]: self.params['SZ'][i] = pgrec['data_size']
|
|
@@ -3549,7 +3331,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
3549
3331
|
wcnd = "wid = {}".format(pgrec['wid'])
|
|
3550
3332
|
if (oflag + wflag) > 0:
|
|
3551
3333
|
locflag = "O" if oflag else "G"
|
|
3552
|
-
lrec = {'locflag'
|
|
3334
|
+
lrec = {'locflag': locflag}
|
|
3553
3335
|
mcnt += self.pgupdt_wfile(dsid, lrec, wcnd, self.LGEREX)
|
|
3554
3336
|
else:
|
|
3555
3337
|
ccnt = self.record_filenumber(dsid, pgrec['gindex'], 4, (pgrec['type'] if pgrec['status'] == 'P' else ''), -1, -pgrec['data_size'])
|
|
@@ -3683,13 +3465,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3683
3465
|
if tmpgs: self.params['GI'] = self.params['OG']
|
|
3684
3466
|
aolds = [None]*self.ALLCNT
|
|
3685
3467
|
solds = [None]*self.ALLCNT
|
|
3686
|
-
oolds = [None]*self.ALLCNT
|
|
3687
3468
|
tolds = [None]*self.ALLCNT
|
|
3688
3469
|
for i in range(self.ALLCNT):
|
|
3689
3470
|
type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'P'
|
|
3690
3471
|
aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
|
|
3691
3472
|
solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
|
|
3692
|
-
oolds[i] = self.get_object_path(solds[i], self.params['DS'])
|
|
3693
3473
|
tolds[i] = type
|
|
3694
3474
|
if tmpds: self.params['DS'] = tmpds
|
|
3695
3475
|
if tmpgs: self.params['GI'] = tmpgs
|
|
@@ -3697,13 +3477,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3697
3477
|
init = 1 if (tmpds or tmpgs) else 0
|
|
3698
3478
|
anews = [None]*self.ALLCNT
|
|
3699
3479
|
snews = [None]*self.ALLCNT
|
|
3700
|
-
onews = [None]*self.ALLCNT
|
|
3701
3480
|
tnews = [None]*self.ALLCNT
|
|
3702
3481
|
for i in range(self.ALLCNT):
|
|
3703
3482
|
type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'P'
|
|
3704
3483
|
anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
|
|
3705
3484
|
snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
|
|
3706
|
-
onews[i] = self.get_object_path(snews[i], dsid)
|
|
3707
3485
|
tnews[i] = type
|
|
3708
3486
|
init = 0
|
|
3709
3487
|
fnames = "FIT"
|
|
@@ -3715,13 +3493,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
3715
3493
|
if bidx == self.ALLCNT: return
|
|
3716
3494
|
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
3717
3495
|
chksize = self.PGLOG['DSCHECK']['size']
|
|
3718
|
-
reorder = self.MODCNT = scnt =
|
|
3496
|
+
reorder = self.MODCNT = scnt = 0
|
|
3719
3497
|
for i in range(bidx, self.ALLCNT):
|
|
3720
3498
|
if i > bidx and ((i-bidx)%20) == 0:
|
|
3721
3499
|
if self.PGLOG['DSCHECK']:
|
|
3722
3500
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
3723
3501
|
if 'EM' in self.params:
|
|
3724
|
-
self.PGLOG['PRGMSG'] = "{}/{}/{}
|
|
3502
|
+
self.PGLOG['PRGMSG'] = "{}/{}/{}, Disk/Record/Proccessed, of {} Saved file{} moved".format(scnt, self.MODCNT, i, self.ALLCNT, s)
|
|
3725
3503
|
type = tolds[i]
|
|
3726
3504
|
pgrec = self.pgget(tname, "*", "sfile = '{}' AND dsid = '{}' AND type = '{}'".format(solds[i], self.params['OD'], type), self.LGEREX)
|
|
3727
3505
|
if not pgrec:
|
|
@@ -3746,27 +3524,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
3746
3524
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3747
3525
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3748
3526
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3749
|
-
|
|
3750
|
-
smove = 1
|
|
3751
|
-
# if pgrec['locflag'] == 'O':
|
|
3752
|
-
# smove = 0
|
|
3753
|
-
# elif pgrec['locflag'] == 'G':
|
|
3754
|
-
# omove = 0
|
|
3755
|
-
if smove and aolds[i] != anews[i]:
|
|
3527
|
+
if aolds[i] != anews[i]:
|
|
3756
3528
|
if not self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE):
|
|
3757
3529
|
self.RETSTAT = 1
|
|
3758
3530
|
continue
|
|
3759
3531
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3760
3532
|
scnt += 1
|
|
3761
|
-
if omove and oolds[i] != onews[i]:
|
|
3762
|
-
if not self.move_object_file(onews[i], oolds[i], bucket, bucket, self.PGOPT['emerol']|self.OVERRIDE):
|
|
3763
|
-
self.RETSTAT = 1
|
|
3764
|
-
continue
|
|
3765
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3766
|
-
ocnt += 1
|
|
3767
3533
|
self.set_one_savedfile(i, pgrec, snews[i], fnames, tnews[i], None, tmpds)
|
|
3768
3534
|
if pgrec['bid']: self.save_move_info(pgrec['bid'], solds[i], type, 'S', self.params['OD'], snews[i], tnews[i], 'S', dsid)
|
|
3769
|
-
self.pglog("{}/{}
|
|
3535
|
+
self.pglog("{}/{}, Disk/Record, of {} Saved file{} moved".format(scnt, self.MODCNT, self.ALLCNT, s), self.PGOPT['emllog'])
|
|
3770
3536
|
if self.PGLOG['DSCHECK']:
|
|
3771
3537
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
3772
3538
|
if 'ON' in self.params:
|
|
@@ -3881,7 +3647,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
3881
3647
|
dsid = self.params['DS']
|
|
3882
3648
|
dcnd = "dsid = '{}'".format(dsid)
|
|
3883
3649
|
frombucket = self.PGLOG['OBJCTBKT']
|
|
3884
|
-
tobucket = "gdex-decsdata"
|
|
3885
3650
|
bidx = chksize = 0
|
|
3886
3651
|
tmpds = tmpgs = None
|
|
3887
3652
|
if 'OD' not in self.params:
|
|
@@ -3922,20 +3687,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
3922
3687
|
init = 1 if (tmpds or tmpgs) else 0
|
|
3923
3688
|
anews = [None]*self.ALLCNT
|
|
3924
3689
|
snews = [None]*self.ALLCNT
|
|
3925
|
-
onews = [None]*self.ALLCNT
|
|
3926
3690
|
tnews = [None]*self.ALLCNT
|
|
3927
3691
|
for i in range(self.ALLCNT):
|
|
3928
3692
|
type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'V'
|
|
3929
3693
|
anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
|
|
3930
3694
|
snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
|
|
3931
|
-
onews[i] = self.join_paths(dsid, snews[i])
|
|
3932
3695
|
tnews[i] = type
|
|
3933
3696
|
init = 0
|
|
3934
3697
|
self.validate_multiple_values(tname, self.ALLCNT)
|
|
3935
3698
|
if self.PGLOG['DSCHECK']:
|
|
3936
3699
|
bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
|
|
3937
3700
|
if bidx > 0:
|
|
3938
|
-
self.pglog("bidx of self.ALLCNT Web to Saved files processed for move", self.PGOPT['emllog'])
|
|
3701
|
+
self.pglog(f"{bidx} of {self.ALLCNT} Web to Saved files processed for move", self.PGOPT['emllog'])
|
|
3939
3702
|
if bidx == self.ALLCNT: return
|
|
3940
3703
|
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
3941
3704
|
chksize = self.PGLOG['DSCHECK']['size']
|
|
@@ -3962,7 +3725,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
3962
3725
|
if self.PGLOG['DSCHECK']:
|
|
3963
3726
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
3964
3727
|
if 'EM' in self.params:
|
|
3965
|
-
self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {}
|
|
3728
|
+
self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {} Web file{} moved".format(wcnt, ocnt, dcnt, self.ADDCNT, i, self.ALLCNT, s)
|
|
3966
3729
|
type = tolds[i]
|
|
3967
3730
|
pgrec = self.pgget_wfile(self.params['OD'], "*", "wfile = '{}' AND type = '{}'".format(wolds[i], type), self.LGEREX)
|
|
3968
3731
|
if not pgrec:
|
|
@@ -3981,45 +3744,23 @@ class DsArch(PgArch, PgMeta):
|
|
|
3981
3744
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3982
3745
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3983
3746
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3984
|
-
ofrom = wfrom =
|
|
3985
|
-
omove = 0
|
|
3747
|
+
ofrom = wfrom = 1
|
|
3986
3748
|
locflag = pgrec['locflag']
|
|
3987
3749
|
if locflag == 'O':
|
|
3988
3750
|
wfrom = 0
|
|
3989
3751
|
elif locflag == 'G':
|
|
3990
3752
|
ofrom = 0
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
|
|
3994
|
-
# locflag = self.params['LC'][i]
|
|
3995
|
-
# if locflag == 'O':
|
|
3996
|
-
# smove = 0
|
|
3997
|
-
# elif locflag == 'G':
|
|
3998
|
-
# omove = 0
|
|
3999
|
-
if smove:
|
|
4000
|
-
if wfrom:
|
|
4001
|
-
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
4002
|
-
wfrom = 0
|
|
4003
|
-
else:
|
|
4004
|
-
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4005
|
-
if not stat:
|
|
4006
|
-
self.RETSTAT = 1
|
|
4007
|
-
continue
|
|
3753
|
+
if wfrom:
|
|
3754
|
+
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
3755
|
+
wfrom = 0
|
|
4008
3756
|
wcnt += 1
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
if wfrom:
|
|
4012
|
-
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4013
|
-
elif smove:
|
|
4014
|
-
stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4015
|
-
else:
|
|
4016
|
-
stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4017
|
-
ofrom = 0
|
|
4018
|
-
if not stat:
|
|
4019
|
-
self.RETSTAT = 1
|
|
4020
|
-
continue
|
|
3757
|
+
else:
|
|
3758
|
+
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4021
3759
|
ocnt += 1
|
|
4022
|
-
|
|
3760
|
+
if not stat:
|
|
3761
|
+
self.RETSTAT = 1
|
|
3762
|
+
continue
|
|
3763
|
+
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4023
3764
|
if wfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
|
|
4024
3765
|
if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
|
|
4025
3766
|
if self.PGOPT['GXTYP'].find(type) > -1 and ('DX' in self.params or pgrec and pgrec['meta_link'] and pgrec['meta_link'] != 'N' and 'KM' not in self.params):
|
|
@@ -4053,7 +3794,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4053
3794
|
|
|
4054
3795
|
# get date/time/size info from given file record
|
|
4055
3796
|
def get_file_origin_info(self, fname, pgrec):
|
|
4056
|
-
info = {'isfile'
|
|
3797
|
+
info = {'isfile': (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size': pgrec['data_size']}
|
|
4057
3798
|
info['fname'] = op.basename(fname)
|
|
4058
3799
|
info['date_modified'] = pgrec['date_modified']
|
|
4059
3800
|
info['time_modified'] = pgrec['time_modified']
|
|
@@ -4066,10 +3807,9 @@ class DsArch(PgArch, PgMeta):
|
|
|
4066
3807
|
tname = 'sfile'
|
|
4067
3808
|
dsid = self.params['DS']
|
|
4068
3809
|
dcnd = "dsid = '{}'".format(dsid)
|
|
4069
|
-
bucket = "gdex-decsdata"
|
|
4070
3810
|
s = 's' if self.ALLCNT > 1 else ''
|
|
4071
3811
|
self.pglog("Delete {} Saved file{} from {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
|
|
4072
|
-
bidx = chksize = reorder = scnt =
|
|
3812
|
+
bidx = chksize = reorder = scnt = dcnt = 0
|
|
4073
3813
|
self.cache_group_info(self.ALLCNT, 0)
|
|
4074
3814
|
self.validate_multiple_options(self.ALLCNT, ["ST", 'VI', 'QF', 'LC'])
|
|
4075
3815
|
if self.PGLOG['DSCHECK']:
|
|
@@ -4084,7 +3824,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4084
3824
|
if self.PGLOG['DSCHECK']:
|
|
4085
3825
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
4086
3826
|
if 'EM' in self.params:
|
|
4087
|
-
self.PGLOG['PRGMSG'] = "{}/{}/
|
|
3827
|
+
self.PGLOG['PRGMSG'] = "{}/{}/i, Disk/Record/processed, of {} Saved file{} deleted".format(scnt, dcnt, i, self.ALLCNT, s)
|
|
4088
3828
|
sfile = self.params['SF'][i]
|
|
4089
3829
|
if 'ST' in self.params and self.params['ST'][i]:
|
|
4090
3830
|
type = self.params['ST'][i]
|
|
@@ -4096,24 +3836,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
4096
3836
|
if not pgrec:
|
|
4097
3837
|
self.pglog("{}-{}: Type '{}' Saved file is not in RDADB".format(dsid, sfile, type), self.PGOPT['errlog'])
|
|
4098
3838
|
continue
|
|
4099
|
-
sdel =
|
|
4100
|
-
|
|
4101
|
-
locflag
|
|
4102
|
-
# locflag = pgrec['locflag']
|
|
4103
|
-
# if locflag == 'O':
|
|
4104
|
-
# sflag = 0
|
|
4105
|
-
# elif locflag == 'G':
|
|
4106
|
-
# oflag = 0
|
|
4107
|
-
# elif locflag == 'C':
|
|
4108
|
-
# sflag = oflag = 0
|
|
3839
|
+
sdel = sflag = 1
|
|
3840
|
+
locflag = pgrec['locflag']
|
|
3841
|
+
if locflag != 'G': sflag = 0
|
|
4109
3842
|
if 'LC' in self.params and self.params['LC'][i]: locflag = self.params['LC'][i]
|
|
4110
|
-
if locflag
|
|
4111
|
-
|
|
4112
|
-
elif locflag == 'G':
|
|
4113
|
-
odel = 0
|
|
4114
|
-
elif locflag == 'C':
|
|
4115
|
-
sdel = odel = 0
|
|
4116
|
-
if (sflag+oflag) == (sdel+odel):
|
|
3843
|
+
if locflag != 'G': sdel = 0
|
|
3844
|
+
if sflag == sdel:
|
|
4117
3845
|
vindex = self.params['VI'][i] if 'VI' in self.params else pgrec['vindex']
|
|
4118
3846
|
if vindex:
|
|
4119
3847
|
self.pglog(sfile + ": Saved file is version controlled, add option -vi 0 to force delete", self.PGOPT['errlog'])
|
|
@@ -4130,24 +3858,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
4130
3858
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4131
3859
|
elif self.check_local_file(afile) is None:
|
|
4132
3860
|
sflag = 0
|
|
4133
|
-
if
|
|
4134
|
-
|
|
4135
|
-
if self.delete_object_file(ofile, bucket, self.PGOPT['emerol']):
|
|
4136
|
-
ocnt += 1
|
|
4137
|
-
oflag = 0
|
|
4138
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4139
|
-
elif self.check_object_file(ofile, bucket) is None:
|
|
4140
|
-
oflag = 0
|
|
4141
|
-
if (oflag + sflag) > 0:
|
|
4142
|
-
locflag = "O" if oflag else "G"
|
|
4143
|
-
self.pgexec("UPDATE sfile SET locflag = '{}' WHERE sid = {}".format(locflag, pgrec['sid']), self.LGEREX)
|
|
3861
|
+
if sflag > 0:
|
|
3862
|
+
self.pgexec("UPDATE sfile SET locflag = 'G' WHERE sid = {}".format(pgrec['sid']), self.LGEREX)
|
|
4144
3863
|
else:
|
|
4145
3864
|
ccnt = self.record_filenumber(dsid, pgrec['gindex'], 8, 'P', -1, -pgrec['data_size'])
|
|
4146
3865
|
fcnt = self.pgdel_sfile("sid = {}".format(pgrec['sid']), self.LGEREX)
|
|
4147
3866
|
if fcnt: dcnt += fcnt
|
|
4148
3867
|
if ccnt: self.save_filenumber(dsid, 8, 1, fcnt)
|
|
4149
|
-
if (scnt +
|
|
4150
|
-
self.pglog("{}/{}
|
|
3868
|
+
if (scnt + dcnt) > 0:
|
|
3869
|
+
self.pglog("{}/{}, Disk/Record, of {} Saved file{} deleted for {}".format(scnt, dcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
4151
3870
|
if self.PGLOG['DSCHECK']:
|
|
4152
3871
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
4153
3872
|
if 'ON' in self.params:
|
|
@@ -4161,7 +3880,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4161
3880
|
tname = 'bfile'
|
|
4162
3881
|
dsid = self.params['DS']
|
|
4163
3882
|
dcnd = "dsid = '{}'".format(dsid)
|
|
4164
|
-
brec = {'bid'
|
|
3883
|
+
brec = {'bid': 0}
|
|
4165
3884
|
bkend = "gdex-quasar"
|
|
4166
3885
|
drend = "gdex-quasar-drdata"
|
|
4167
3886
|
s = 's' if self.ALLCNT > 1 else ''
|
|
@@ -4244,7 +3963,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4244
3963
|
metacnt = prdcnt = savedcnt = webcnt = rccnt = pgcnt = modcnt = 0
|
|
4245
3964
|
twcnt = tscnt = 0
|
|
4246
3965
|
for i in range(self.ALLCNT):
|
|
4247
|
-
record = {'gindex'
|
|
3966
|
+
record = {'gindex': self.params['GI'][i]}
|
|
4248
3967
|
gcnd = "gindex = {}".format(self.params['OG'][i])
|
|
4249
3968
|
cnd = "{} AND {}".format(dcnd, gcnd)
|
|
4250
3969
|
modcnt += self.pgupdt(tname, record, cnd, self.LGEREX)
|
|
@@ -4257,12 +3976,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
4257
3976
|
savedcnt += self.pgupdt("sfile", record, cnd, self.LGEREX)
|
|
4258
3977
|
rccnt += self.pgupdt("rcrqst", record, cnd, self.LGEREX)
|
|
4259
3978
|
if tgroups[i]:
|
|
4260
|
-
tgrec = {'tindex'
|
|
3979
|
+
tgrec = {'tindex': self.params['GI'][i]}
|
|
4261
3980
|
tcnd = "tindex = {}".format(self.params['OG'][i])
|
|
4262
3981
|
cnd = "{} AND {}".format(dcnd, tcnd)
|
|
4263
3982
|
twcnt += self.pgupdt_wfile(dsid, tgrec, tcnd, self.LGEREX)
|
|
4264
3983
|
tscnt += self.pgupdt("sfile", tgrec, cnd, self.LGEREX)
|
|
4265
|
-
pgrec = {'pindex'
|
|
3984
|
+
pgrec = {'pindex': self.params['GI'][i]}
|
|
4266
3985
|
cnd = "pindex = {} AND {}".format(self.params['OG'][i], dcnd)
|
|
4267
3986
|
pgcnt += self.pgupdt(tname, pgrec, cnd, self.LGEREX)
|
|
4268
3987
|
self.pglog("{} of {} group{} changed".format(modcnt, self.ALLCNT, s), self.LOGWRN)
|
|
@@ -4296,7 +4015,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4296
4015
|
cnt = 0
|
|
4297
4016
|
getkeys = 1
|
|
4298
4017
|
if cnt > 0:
|
|
4299
|
-
values = {'okeys'
|
|
4018
|
+
values = {'okeys': [], 'value': []}
|
|
4300
4019
|
for i in range(cnt):
|
|
4301
4020
|
pgrec = self.pgget("dsokeys", "value", "{}AND okey = '{}'".fomrat(cond, kvalues[i]), self.LGWNEX)
|
|
4302
4021
|
if pgrec:
|
|
@@ -4343,7 +4062,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4343
4062
|
pgrec['value'] = value
|
|
4344
4063
|
mcnt += self.pgupdt("dsokeys", pgrec, cond, self.LGWNEX)
|
|
4345
4064
|
else:
|
|
4346
|
-
pgrec = {'dsid'
|
|
4065
|
+
pgrec = {'dsid': dsid, 'okey': key, 'value': value}
|
|
4347
4066
|
acnt += self.pgadd("dsokeys", pgrec, self.LGWNEX)
|
|
4348
4067
|
self.pglog("{}/{}/{} of {} key/value pairs added/modified/deleted for {}!".format(acnt, mcnt, dcnt, cnt, dsid), self.LOGWRN)
|
|
4349
4068
|
return (acnt + mcnt + dcnt)
|
|
@@ -4353,8 +4072,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
4353
4072
|
date = self.curdate()
|
|
4354
4073
|
cond = "wid = {} and date = '{}'".format(pgrec['wid'], date)
|
|
4355
4074
|
if not self.pgget("wmove", "", cond, self.LGWNEX):
|
|
4356
|
-
record = {'dsid'
|
|
4357
|
-
'wfile'
|
|
4075
|
+
record = {'dsid': pgrec['dsid'], 'uid': self.PGOPT['UID'],
|
|
4076
|
+
'wfile': pgrec['wfile'], 'wid': pgrec['wid'], 'date': date}
|
|
4358
4077
|
self.pgadd("wmove", record, self.LGWNEX)
|
|
4359
4078
|
|
|
4360
4079
|
# reset file counts for saved groups
|
|
@@ -4375,10 +4094,10 @@ class DsArch(PgArch, PgMeta):
|
|
|
4375
4094
|
if 'GI' in self.params:
|
|
4376
4095
|
for gindex in self.params['GI']:
|
|
4377
4096
|
if gindex is None or gindex in cgidxs: continue
|
|
4378
|
-
tcnt += self.
|
|
4097
|
+
tcnt += self.reset_top_gindex(dsid, gindex, act)
|
|
4379
4098
|
cgidxs[gindex] = gindex
|
|
4380
4099
|
else:
|
|
4381
|
-
tcnt += self.
|
|
4100
|
+
tcnt += self.reset_top_gindex(dsid, 0, act)
|
|
4382
4101
|
return tcnt
|
|
4383
4102
|
|
|
4384
4103
|
# set the re-archived file counts for groups
|
|
@@ -4589,7 +4308,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4589
4308
|
addcnt += 1
|
|
4590
4309
|
if not doi: vidx = 0
|
|
4591
4310
|
if vidx:
|
|
4592
|
-
vrec = {'vindex'
|
|
4311
|
+
vrec = {'vindex': vidx}
|
|
4593
4312
|
vcnd = "type = 'D' AND vindex = 0"
|
|
4594
4313
|
fcnt = self.pgupdt_wfile(dsid, vrec, vcnd, self.PGOPT['extlog'])
|
|
4595
4314
|
if fcnt > 0:
|
|
@@ -4641,7 +4360,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4641
4360
|
self.pglog("{}: Cannot terminate for DOI {} is asscoated to Version control {}".format(vinfo, pgrec['doi'], orec['vindex']), self.LOGERR)
|
|
4642
4361
|
continue
|
|
4643
4362
|
doicnt += 1
|
|
4644
|
-
record = {'status'
|
|
4363
|
+
record = {'status': "H"}
|
|
4645
4364
|
record['end_date'] = self.params['ED'][i] if 'ED' in self.params else self.curdate()
|
|
4646
4365
|
record['end_time'] = self.params['ET'][i] if 'ET' in self.params else self.curtime()
|
|
4647
4366
|
if self.pgupdt("dsvrsn", record, cnd, self.PGOPT['extlog']):
|
rda_python_dsarch/pg_meta.py
CHANGED
|
@@ -679,7 +679,7 @@ class PgMeta(PgCMD, PgSplit):
|
|
|
679
679
|
pgrecs = self.pgmget("dsgroup", "gindex", pcnd, self.LGEREX)
|
|
680
680
|
cnt = len(pgrecs['gindex']) if pgrecs else 0
|
|
681
681
|
for i in range(cnt):
|
|
682
|
-
tcnt += self.reset_top_gindex(dsid, pgrecs['gindex'][i])
|
|
682
|
+
tcnt += self.reset_top_gindex(dsid, pgrecs['gindex'][i], act)
|
|
683
683
|
return tcnt
|
|
684
684
|
|
|
685
685
|
# reset metalink in table wfile
|
|
@@ -2,13 +2,13 @@ rda_python_dsarch/PgArch.py,sha256=d1Y87MT49tgQygLAffNff9Qs4c12KXspYyDG8liSnpQ,5
|
|
|
2
2
|
rda_python_dsarch/PgMeta.py,sha256=FV4GytgnJK00FUlyM6iHqpNd7-XTVHO1j5tZtcLTWgY,26845
|
|
3
3
|
rda_python_dsarch/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
4
|
rda_python_dsarch/ds_arch.py,sha256=6bR0y6_rhIXyvbuYd-oZfL1YcaHhWBkojjzYz1yCrVg,233810
|
|
5
|
-
rda_python_dsarch/dsarch.py,sha256=
|
|
5
|
+
rda_python_dsarch/dsarch.py,sha256=dN86W98fU6vbp8DnOMpTxVkA-NTvmxLVlaMiHFfo15Y,232945
|
|
6
6
|
rda_python_dsarch/dsarch.usg,sha256=vsxGneGCIyzqZx5mwQ8xkTmdNmuny56LgokgDkWt5xE,132034
|
|
7
7
|
rda_python_dsarch/pg_arch.py,sha256=Ky_RfY8q0Uq7OjK1UC8npkW8v_eyk_aAN7bypecYP-w,64598
|
|
8
|
-
rda_python_dsarch/pg_meta.py,sha256=
|
|
9
|
-
rda_python_dsarch-2.0.
|
|
10
|
-
rda_python_dsarch-2.0.
|
|
11
|
-
rda_python_dsarch-2.0.
|
|
12
|
-
rda_python_dsarch-2.0.
|
|
13
|
-
rda_python_dsarch-2.0.
|
|
14
|
-
rda_python_dsarch-2.0.
|
|
8
|
+
rda_python_dsarch/pg_meta.py,sha256=f_7w9UxhJk1i-QfkmDvNCPrrDYGcOkNPNImXLNZw0sA,29548
|
|
9
|
+
rda_python_dsarch-2.0.3.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
10
|
+
rda_python_dsarch-2.0.3.dist-info/METADATA,sha256=qZq3KrenH-LV_Np3E0NtnoM93SrEvQNiTE06qVy65vo,670
|
|
11
|
+
rda_python_dsarch-2.0.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
12
|
+
rda_python_dsarch-2.0.3.dist-info/entry_points.txt,sha256=GR7ShgBv8qixS2l4ufAu-jw3N09pbo9zRewwPHqaOxo,60
|
|
13
|
+
rda_python_dsarch-2.0.3.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
|
|
14
|
+
rda_python_dsarch-2.0.3.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|