rda-python-dsarch 2.0.2__py3-none-any.whl → 2.0.4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_dsarch/dsarch.py +61 -341
- rda_python_dsarch/pg_arch.py +1 -1
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/METADATA +1 -1
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/RECORD +8 -8
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/WHEEL +0 -0
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/entry_points.txt +0 -0
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/licenses/LICENSE +0 -0
- {rda_python_dsarch-2.0.2.dist-info → rda_python_dsarch-2.0.4.dist-info}/top_level.txt +0 -0
rda_python_dsarch/dsarch.py
CHANGED
|
@@ -77,13 +77,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
77
77
|
elif self.PGOPT['ACTS'] == self.OPTS['AS'][0]:
|
|
78
78
|
self.ALLCNT = len(self.params['SF']) if 'SF' in self.params else len(self.params['LF'])
|
|
79
79
|
self.cache_group_info(self.ALLCNT)
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
elif 'XM' in self.params:
|
|
83
|
-
self.crosscopy_saved_files('Move')
|
|
84
|
-
else:
|
|
85
|
-
self.archive_saved_files()
|
|
86
|
-
if 'CL' in self.params: self.delete_local_files()
|
|
80
|
+
self.archive_saved_files()
|
|
81
|
+
if 'CL' in self.params: self.delete_local_files()
|
|
87
82
|
elif self.PGOPT['ACTS'] == self.OPTS['AW'][0]:
|
|
88
83
|
self.ALLCNT = len(self.params['WF']) if 'WF' in self.params else len(self.params['LF'])
|
|
89
84
|
if 'ML' in self.params and 'GX' in self.params: del self.params['ML']
|
|
@@ -726,13 +721,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
726
721
|
# archive save files
|
|
727
722
|
def archive_saved_files(self):
|
|
728
723
|
tname = 'sfile'
|
|
729
|
-
dftloc =
|
|
724
|
+
dftloc = 'G'
|
|
730
725
|
dsid = self.params['DS']
|
|
731
726
|
dcnd = "dsid = '{}'".format(dsid)
|
|
732
727
|
s = 's' if self.ALLCNT > 1 else ''
|
|
733
728
|
bidx = chksize = 0
|
|
734
729
|
dflags = {}
|
|
735
|
-
bucket = "gdex-decsdata" # object store bucket
|
|
736
730
|
self.pglog("Archive {} Saved file{} of {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
|
|
737
731
|
if 'ZD' in self.params or 'UZ' in self.params:
|
|
738
732
|
self.compress_localfile_list(self.PGOPT['CACT'], self.ALLCNT)
|
|
@@ -752,18 +746,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
752
746
|
self.params['AF'] = [None]*self.ALLCNT
|
|
753
747
|
self.OPTS['AF'][2] |= 2
|
|
754
748
|
if 'LC' not in self.params:
|
|
755
|
-
self.params['LC'] = [
|
|
749
|
+
self.params['LC'] = [dftloc]*self.ALLCNT
|
|
756
750
|
self.OPTS['LC'][2] |= 2
|
|
757
751
|
if 'SZ' not in self.params:
|
|
758
752
|
self.params['SZ'] = [0]*self.ALLCNT
|
|
759
753
|
self.OPTS['SZ'][2] |= 2
|
|
760
754
|
if 'MC' not in self.params: self.params['MC'] = [None]*self.ALLCNT
|
|
761
|
-
reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt =
|
|
755
|
+
reorder = errcnt = self.ADDCNT = self.MODCNT = bgcnt = acnt = 0
|
|
762
756
|
perrcnt = self.ALLCNT
|
|
763
757
|
efiles = [1]*self.ALLCNT
|
|
764
758
|
if self.PGSIG['BPROC'] > 1:
|
|
765
759
|
afiles = [None]*self.ALLCNT
|
|
766
|
-
ofiles = [None]*self.ALLCNT
|
|
767
760
|
lfiles = [None]*self.ALLCNT
|
|
768
761
|
fnames = None
|
|
769
762
|
override = self.OVERRIDE
|
|
@@ -789,19 +782,9 @@ class DsArch(PgArch, PgMeta):
|
|
|
789
782
|
self.params['LF'][i] = self.params['SF'][i] = None
|
|
790
783
|
continue
|
|
791
784
|
locflag = self.params['LC'][i]
|
|
792
|
-
if not locflag or locflag == 'R':
|
|
793
|
-
if not dftloc: dftloc = self.get_dataset_locflag(dsid)
|
|
794
|
-
locflag = self.params['LC'][i] = dftloc
|
|
795
785
|
if locflag == 'C': self.pglog(lfile + ": Cannot Archive Saved File for CGD data", self.PGOPT['extlog'])
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
elif locflag == 'O':
|
|
799
|
-
oarch = 1
|
|
800
|
-
sarch = 0
|
|
801
|
-
else:
|
|
802
|
-
oarch = 0
|
|
803
|
-
sarch = 1
|
|
804
|
-
if oarch: self.pglog(lfile + ": Cannot Archive Saved File onto Boreas", self.PGOPT['extlog'])
|
|
786
|
+
locflag = self.params['LC'][i] = dftloc
|
|
787
|
+
sarch = 1
|
|
805
788
|
if not self.params['MC'][i]: self.params['MC'][i] = self.get_md5sum(lfile) # re-get MD5 Checksum
|
|
806
789
|
if not (self.params['SZ'][i] and self.params['SZ'][i] == lsize):
|
|
807
790
|
self.params['SZ'][i] = lsize
|
|
@@ -817,9 +800,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
817
800
|
else:
|
|
818
801
|
self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['extlog'])
|
|
819
802
|
continue
|
|
820
|
-
afile =
|
|
803
|
+
afile = self.get_saved_path(i, sfile, 1, type)
|
|
821
804
|
sfile = self.get_saved_path(i, sfile, 0, type)
|
|
822
|
-
ofile = (self.join_paths(dsid, sfile) if oarch else None)
|
|
823
805
|
pgrec = self.pgget(tname, "*", "{} AND sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
|
|
824
806
|
if not pgrec:
|
|
825
807
|
pgrec = self.pgget(tname, "type", "{} AND sfile = '{}'".format(dcnd, sfile), self.PGOPT['extlog'])
|
|
@@ -851,45 +833,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
851
833
|
efiles[i] = 1
|
|
852
834
|
dflags['G'] = self.PGLOG['DECSHOME']
|
|
853
835
|
continue
|
|
854
|
-
if
|
|
855
|
-
|
|
856
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
857
|
-
if info:
|
|
858
|
-
if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
|
|
859
|
-
self.pglog("Object-{}-{}: Same-Checksum ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['emllog'])
|
|
860
|
-
oarch = 0
|
|
861
|
-
elif info['data_size'] == lsize:
|
|
862
|
-
self.pglog("Object-{}-{}: Same-Size ARCHIVED at {}:{}".format(bucket, ofile, info['date_modified'], info['time_modified']), self.PGOPT['wrnlog'])
|
|
863
|
-
oarch = 0
|
|
864
|
-
elif vsnctl and not override:
|
|
865
|
-
self.pglog("Object-{}-{}: Cannot rearchive version controlled file".format(bucket, ofile), self.PGOPT['extlog'])
|
|
866
|
-
self.params['SF'][i] = None
|
|
867
|
-
continue
|
|
868
|
-
else:
|
|
869
|
-
replace = 1
|
|
870
|
-
elif info is not None:
|
|
836
|
+
if sarch:
|
|
837
|
+
if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
|
|
871
838
|
errcnt += 1
|
|
872
839
|
efiles[i] = 1
|
|
873
|
-
dflags['O'] = bucket
|
|
874
840
|
continue
|
|
875
|
-
|
|
876
|
-
if sarch:
|
|
877
|
-
if not self.local_copy_local(afile, lfile, self.PGOPT['emerol']|override):
|
|
878
|
-
errcnt += 1
|
|
879
|
-
efiles[i] = 1
|
|
880
|
-
continue
|
|
881
|
-
acnt += 1
|
|
882
|
-
if oarch:
|
|
883
|
-
if replace: self.delete_object_file(ofile, bucket)
|
|
884
|
-
if not self.local_copy_object(ofile, lfile, bucket, None, self.PGOPT['emerol']|override):
|
|
885
|
-
errcnt += 1
|
|
886
|
-
efiles[i] = 1
|
|
887
|
-
continue
|
|
888
|
-
ocnt += 1
|
|
841
|
+
acnt += 1
|
|
889
842
|
if self.PGLOG['DSCHECK']: chksize += lsize
|
|
890
843
|
if self.PGSIG['BPROC'] > 1:
|
|
891
844
|
afiles[i] = afile
|
|
892
|
-
ofiles[i] = ofile
|
|
893
845
|
lfiles[i] = lfile
|
|
894
846
|
bgcnt += 1
|
|
895
847
|
if self.PGSIG['BPROC'] < 2:
|
|
@@ -897,12 +849,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
897
849
|
info = None
|
|
898
850
|
if sarch:
|
|
899
851
|
info = self.check_local_file(afile, 1, self.PGOPT['emerol']|self.PFSIZE)
|
|
900
|
-
elif oarch:
|
|
901
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
902
852
|
elif pgrec:
|
|
903
853
|
info = self.get_file_origin_info(sfile, pgrec)
|
|
904
|
-
elif locflag == 'O':
|
|
905
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
906
854
|
sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
|
|
907
855
|
if not sid:
|
|
908
856
|
self.params['LF'][i] = self.params['SF'][i] = None
|
|
@@ -922,11 +870,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
922
870
|
self.check_background(None, 0, self.LOGWRN, 1)
|
|
923
871
|
for i in range(self.ALLCNT):
|
|
924
872
|
if afiles[i]: self.validate_gladearch(afiles[i], lfiles[i], i)
|
|
925
|
-
if ofiles[i]: self.validate_objectarch(ofiles[i], lfiles[i], bucket, i)
|
|
926
873
|
if acnt > 0:
|
|
927
874
|
self.pglog("{} of {} Saved file{} archived for {}".format(acnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
928
|
-
if ocnt > 0:
|
|
929
|
-
self.pglog("{] of {} Object file{} archived for {}".format(ocnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
930
875
|
if self.PGLOG['DSCHECK']:
|
|
931
876
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
932
877
|
self.pglog("{}/{} of {} Saved file record{} added/modified for {}!".format(self.ADDCNT, self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
@@ -1230,147 +1175,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
1230
1175
|
self.reset_rdadb_version(dsid)
|
|
1231
1176
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1232
1177
|
|
|
1233
|
-
# cross copy save files between glade and object store
|
|
1234
|
-
def crosscopy_saved_files(self, aname):
|
|
1235
|
-
tname = 'sfile'
|
|
1236
|
-
dsid = self.params['DS']
|
|
1237
|
-
dcnd = "dsid = '{}'".format(dsid)
|
|
1238
|
-
s = 's' if self.ALLCNT > 1 else ''
|
|
1239
|
-
bidx = chksize = 0
|
|
1240
|
-
dflags = {}
|
|
1241
|
-
bucket = "gdex-decsdata"
|
|
1242
|
-
self.pglog("Cross {} {} Saved file{} of {} ...".format(aname, self.ALLCNT, s, dsid), self.WARNLG)
|
|
1243
|
-
self.validate_multiple_values(tname, self.ALLCNT)
|
|
1244
|
-
if self.PGLOG['DSCHECK']:
|
|
1245
|
-
bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
|
|
1246
|
-
if bidx > 0:
|
|
1247
|
-
self.pglog("{} of {} file{} processed for SAVED archive".format(bidx, self.ALLCNT, s), self.PGOPT['emllog'])
|
|
1248
|
-
if bidx == self.ALLCNT: return
|
|
1249
|
-
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
1250
|
-
chksize = self.PGLOG['DSCHECK']['size']
|
|
1251
|
-
reorder = errcnt = self.MODCNT = bgcnt = acnt = ocnt = 0
|
|
1252
|
-
perrcnt = self.ALLCNT
|
|
1253
|
-
efiles = [1]*self.ALLCNT
|
|
1254
|
-
self.params['LC'] = ['B']*self.ALLCNT
|
|
1255
|
-
if self.PGSIG['BPROC'] > 1:
|
|
1256
|
-
afiles = [None]*self.ALLCNT
|
|
1257
|
-
ofiles = [None]*self.ALLCNT
|
|
1258
|
-
sarchs = [None]*self.ALLCNT
|
|
1259
|
-
fnames = None
|
|
1260
|
-
while True:
|
|
1261
|
-
for i in range(bidx, self.ALLCNT):
|
|
1262
|
-
if self.PGSIG['BPROC'] < 2 and i > bidx and ((i-bidx)%20) == 0:
|
|
1263
|
-
if self.PGLOG['DSCHECK']:
|
|
1264
|
-
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
1265
|
-
if 'EM' in self.params:
|
|
1266
|
-
self.PGLOG['PRGMSG'] = "{}/{} of {} saved file{} archived/processed".format(acnt, i, self.ALLCNT, s)
|
|
1267
|
-
sfile = self.params['SF'][i]
|
|
1268
|
-
if not (efiles[i] and sfile): continue
|
|
1269
|
-
efiles[i] = 0
|
|
1270
|
-
if 'ST' in self.params and self.params['ST'][i]:
|
|
1271
|
-
type = self.params['ST'][i]
|
|
1272
|
-
if self.PGOPT['SDTYP'].find(type) < 0:
|
|
1273
|
-
self.pglog("{}-{}: Invalid Saved file Type '{}' to Archive".format(dsid, sfile, type), self.PGOPT['emerol'])
|
|
1274
|
-
continue
|
|
1275
|
-
else:
|
|
1276
|
-
self.pglog("{}-{}: Miss Saved file Type to Archive".format(dsid, sfile), self.PGOPT['errlog'])
|
|
1277
|
-
continue
|
|
1278
|
-
sfile = self.get_saved_path(i, sfile, 0, type)
|
|
1279
|
-
afile = self.get_saved_path(i, sfile, 1, type)
|
|
1280
|
-
ofile = self.join_paths(dsid, sfile)
|
|
1281
|
-
sinfo = "{}-{}-{}".format(dsid, type, sfile)
|
|
1282
|
-
pgrec = self.pgget(tname, "*", "{} and sfile = '{}' AND type = '{}'".format(dcnd, sfile, type), self.PGOPT['extlog'])
|
|
1283
|
-
if not pgrec:
|
|
1284
|
-
self.pglog("{}: Fail to Cross {} for Saved file not in RDADB".format(sinfo, aname), self.PGOPT['emlerr'])
|
|
1285
|
-
continue
|
|
1286
|
-
elif pgrec['locflag'] == 'C':
|
|
1287
|
-
self.pglog("{}: Fail to Cross {} Saved File for CGD data".format(sinfo, aname), self.PGOPT['extlog'])
|
|
1288
|
-
if pgrec and self.params['SF'][i] != sfile: self.params['SF'][i] = sfile
|
|
1289
|
-
sarch = oarch = 1
|
|
1290
|
-
self.pglog(sinfo + ": Cross {} Saved file ...".format(aname), self.WARNLG)
|
|
1291
|
-
info = self.check_local_file(afile, 0, self.PGOPT['emerol']|self.PFSIZE)
|
|
1292
|
-
if info:
|
|
1293
|
-
sarch = 0
|
|
1294
|
-
elif info is not None:
|
|
1295
|
-
errcnt += 1
|
|
1296
|
-
efiles[i] = 1
|
|
1297
|
-
dflags['G'] = self.PGLOG['DECSHOME']
|
|
1298
|
-
continue
|
|
1299
|
-
info = self.check_object_file(ofile, bucket, 1, self.PGOPT['emerol'])
|
|
1300
|
-
if info:
|
|
1301
|
-
oarch = 0
|
|
1302
|
-
elif info is not None:
|
|
1303
|
-
errcnt += 1
|
|
1304
|
-
efiles[i] = 1
|
|
1305
|
-
dflags['O'] = bucket
|
|
1306
|
-
continue
|
|
1307
|
-
if sarch and oarch:
|
|
1308
|
-
self.pglog(sinfo + ": Cannot Cross {}, Neither Saved Nor Object file Exists".format(aname), self.PGOPT['errlog'])
|
|
1309
|
-
continue
|
|
1310
|
-
elif not (sarch or oarch) and pgrec['locflag'] == 'B':
|
|
1311
|
-
self.pglog(sinfo + ": No need Cross {}, Both Saved & Object Exist".format(aname), self.PGOPT['wrnlog'])
|
|
1312
|
-
continue
|
|
1313
|
-
if sarch:
|
|
1314
|
-
if not self.object_copy_local(afile, ofile, bucket, self.PGOPT['emerol']|self.OVERRIDE):
|
|
1315
|
-
errcnt += 1
|
|
1316
|
-
efiles[i] = 1
|
|
1317
|
-
dflags['G'] = self.PGLOG['DECSHOME']
|
|
1318
|
-
continue
|
|
1319
|
-
if aname == 'Move':
|
|
1320
|
-
self.params['LC'][i] = 'G'
|
|
1321
|
-
self.delete_object_file(ofile, bucket, self.PGOPT['extlog'])
|
|
1322
|
-
acnt += 1
|
|
1323
|
-
elif oarch:
|
|
1324
|
-
if not self.local_copy_object(ofile, afile, bucket, None, self.PGOPT['emerol']|self.OVERRIDE):
|
|
1325
|
-
errcnt += 1
|
|
1326
|
-
efiles[i] = 1
|
|
1327
|
-
dflags['O'] = bucket
|
|
1328
|
-
continue
|
|
1329
|
-
if aname == 'Move':
|
|
1330
|
-
self.params['LC'][i] = 'O'
|
|
1331
|
-
self.delete_local_file(afile, self.PGOPT['extlog'])
|
|
1332
|
-
ocnt += 1
|
|
1333
|
-
if self.PGSIG['BPROC'] > 1:
|
|
1334
|
-
afiles[i] = afile
|
|
1335
|
-
ofiles[i] = ofile
|
|
1336
|
-
sarchs[i] = sarch
|
|
1337
|
-
bgcnt += 1
|
|
1338
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
1339
|
-
if self.PGSIG['BPROC'] < 2:
|
|
1340
|
-
if not fnames: fnames = self.get_field_keys(tname, None, "G") # get setting fields if not yet
|
|
1341
|
-
info = self.get_file_origin_info(sfile, pgrec)
|
|
1342
|
-
sid = self.set_one_savedfile(i, pgrec, sfile, fnames, type, info)
|
|
1343
|
-
if not sid:
|
|
1344
|
-
self.params['SF'][i] = None
|
|
1345
|
-
continue
|
|
1346
|
-
if errcnt == 0 or errcnt >= perrcnt or self.PGLOG['DSCHECK']: break
|
|
1347
|
-
perrcnt = errcnt
|
|
1348
|
-
self.pglog("Recopy {} Saved file{} for {}".format(errcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
1349
|
-
errcnt = self.reset_errmsg(0)
|
|
1350
|
-
if errcnt:
|
|
1351
|
-
self.RETSTAT = self.reset_errmsg(errcnt)
|
|
1352
|
-
for i in range(bidx, self.ALLCNT):
|
|
1353
|
-
if efiles[i]: self.params['SF'][i] = ''
|
|
1354
|
-
if self.PGLOG['DSCHECK']:
|
|
1355
|
-
self.check_storage_dflags(dflags, self.PGLOG['DSCHECK'], self.PGOPT['emerol'])
|
|
1356
|
-
if bgcnt:
|
|
1357
|
-
self.check_background(None, 0, self.LOGWRN, 1)
|
|
1358
|
-
for i in range(self.ALLCNT):
|
|
1359
|
-
if sarchs[i]:
|
|
1360
|
-
self.validate_gladearch(afiles[i], "{}-{}".format(bucket, ofiles[i]), i)
|
|
1361
|
-
elif ofiles[i]:
|
|
1362
|
-
self.validate_objectarch(ofiles[i], afiles[i], bucket, i)
|
|
1363
|
-
astr = 'Moved' if aname == 'Move' else 'Copied'
|
|
1364
|
-
if acnt > 0: self.pglog("{} of {} Saved file{} Cross {} for {}".format(acnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
|
|
1365
|
-
if ocnt > 0: self.pglog("{} of {} Object file{} Cross {} for {}".format(ocnt, self.ALLCNT, s, astr, dsid), self.PGOPT['emllog'])
|
|
1366
|
-
if self.PGLOG['DSCHECK']:
|
|
1367
|
-
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
1368
|
-
self.pglog("{} of {} Saved file record{} modified for {}!".format(self.MODCNT, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
1369
|
-
if 'ON' in self.params: reorder = self.reorder_files(dsid, self.params['ON'], tname)
|
|
1370
|
-
if (self.MODCNT + reorder) > 0:
|
|
1371
|
-
self.reset_rdadb_version(dsid)
|
|
1372
|
-
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1373
|
-
|
|
1374
1178
|
# get backup file names in RDADB and on Quaser server for given dataset id
|
|
1375
1179
|
def get_backup_filenames(self, bfile, dsid):
|
|
1376
1180
|
ms = re.match(r'^/{}/(.+)$'.format(dsid), bfile)
|
|
@@ -1421,7 +1225,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
1421
1225
|
ifidx += 1
|
|
1422
1226
|
ccnt *= 3
|
|
1423
1227
|
self.set_dscheck_fcount(ccnt, self.PGOPT['extlog'])
|
|
1424
|
-
tinfo = {'bid'
|
|
1228
|
+
tinfo = {'bid': 0, 'size': 0, 'cnt': 0, 'afmt': '', 'dfmt': '', 'sids': [], 'wids': []}
|
|
1425
1229
|
if pgbck: tinfo['bid'] = pgbck['bid']
|
|
1426
1230
|
ifidx = 1
|
|
1427
1231
|
while True:
|
|
@@ -1434,13 +1238,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
1434
1238
|
ifidx += 1
|
|
1435
1239
|
info = self.check_local_file(tarfile, 33) # 1+32
|
|
1436
1240
|
fsize = info['data_size'] if info else 0
|
|
1437
|
-
# if fsize < tinfo['size']:
|
|
1438
|
-
# self.pglog("{}: Backup file size {} is less than total file size {}".format(tarfile, fsize, tinfo['size']), self.PGOPT['extlog'])
|
|
1439
1241
|
if fsize < self.PGLOG['ONEGBS']:
|
|
1440
1242
|
self.pglog("{}: Backup file size {} is less than one GB".format(tarfile, fsize), self.PGOPT['extlog'])
|
|
1441
|
-
record = {'type'
|
|
1442
|
-
'uid'
|
|
1443
|
-
'scount'
|
|
1243
|
+
record = {'type': qtype, 'data_format': tinfo['dfmt'], 'data_size': fsize,
|
|
1244
|
+
'uid': self.PGOPT['UID'], 'checksum': info['checksum'],
|
|
1245
|
+
'scount': scnt, 'wcount': wcnt}
|
|
1444
1246
|
record['file_format'] = self.append_format_string(tinfo['afmt'], tfmt, 1)
|
|
1445
1247
|
record['date_created'] = record['date_modified'] = info['date_modified']
|
|
1446
1248
|
record['time_created'] = record['time_modified'] = info['time_modified']
|
|
@@ -1469,7 +1271,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
1469
1271
|
self.pglog("{}: Error add Quaser Backup file name in RDADB for {}".format(bfile, dsid), self.PGOPT['extlog'])
|
|
1470
1272
|
tcnt = tinfo['cnt']
|
|
1471
1273
|
tsize = tinfo['size']
|
|
1472
|
-
brec = {'bid'
|
|
1274
|
+
brec = {'bid': bid}
|
|
1473
1275
|
if scnt:
|
|
1474
1276
|
for sid in tinfo['sids']:
|
|
1475
1277
|
tcnt += self.pgupdt("sfile", brec, "sid = {}".format(sid))
|
|
@@ -2261,7 +2063,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
2261
2063
|
cnd = "{} AND {}".format(dcnd, gcnd)
|
|
2262
2064
|
pgrecs = self.pgmget_wfile(dsid, 'wfile, type', gcnd, self.PGOPT['extlog'])
|
|
2263
2065
|
cnt = len(pgrecs['wfile']) if pgrecs else 0
|
|
2264
|
-
srec = {'status'
|
|
2066
|
+
srec = {'status': 'I'}
|
|
2265
2067
|
if cnt:
|
|
2266
2068
|
s = 's' if cnt > 1 else ''
|
|
2267
2069
|
self.pglog("{}: set {} web file{} to Internal".format(grp, cnt, s), self.PGOPT['wrnlog'])
|
|
@@ -2609,7 +2411,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
2609
2411
|
if 'status' in record: self.change_wfile_mode(dsid, file, type, pgrec['status'], record['status'])
|
|
2610
2412
|
if mlink or pgrec['meta_link'] and pgrec['meta_link'] == 'Y': self.set_meta_link(dsid, file)
|
|
2611
2413
|
else:
|
|
2612
|
-
if record['wfile'] != file: record['wfile'] = file
|
|
2414
|
+
if 'wfile' not in record or record['wfile'] != file: record['wfile'] = file
|
|
2613
2415
|
record['uid'] = self.PGOPT['UID']
|
|
2614
2416
|
if 'status' not in record: record['status'] = 'P'
|
|
2615
2417
|
if not info:
|
|
@@ -3289,7 +3091,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
3289
3091
|
tname = 'wfile'
|
|
3290
3092
|
dsid = self.params['DS']
|
|
3291
3093
|
dcnd = "dsid = '{}'".format(dsid)
|
|
3292
|
-
frombucket = "gdex-decsdata"
|
|
3293
3094
|
tobucket = self.PGLOG['OBJCTBKT']
|
|
3294
3095
|
s = 's' if self.ALLCNT > 1 else ''
|
|
3295
3096
|
bidx = chksize = 0
|
|
@@ -3318,13 +3119,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3318
3119
|
if tmpgs: self.params['GI'] = self.params['OG']
|
|
3319
3120
|
aolds = [None]*self.ALLCNT
|
|
3320
3121
|
solds = [None]*self.ALLCNT
|
|
3321
|
-
oolds = [None]*self.ALLCNT
|
|
3322
3122
|
tolds = [None]*self.ALLCNT
|
|
3323
3123
|
for i in range(self.ALLCNT):
|
|
3324
3124
|
type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'V'
|
|
3325
3125
|
aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
|
|
3326
3126
|
solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
|
|
3327
|
-
oolds[i] = self.join_paths(self.params['DS'], solds[i])
|
|
3328
3127
|
tolds[i] = type
|
|
3329
3128
|
if tmpds: self.params['DS'] = tmpds
|
|
3330
3129
|
if tmpgs: self.params['GI'] = tmpgs
|
|
@@ -3391,14 +3190,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
3391
3190
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3392
3191
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3393
3192
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3394
|
-
|
|
3395
|
-
ofrom = 0
|
|
3193
|
+
omove = wmove = sfrom = 1
|
|
3396
3194
|
locflag = 'G'
|
|
3397
|
-
# locflag = pgrec['locflag']
|
|
3398
|
-
# if locflag == 'O':
|
|
3399
|
-
# sfrom = 0
|
|
3400
|
-
# elif locflag == 'G':
|
|
3401
|
-
# ofrom = 0
|
|
3402
3195
|
if not self.params['LC'][i] or self.params['LC'][i] == 'R':
|
|
3403
3196
|
self.params['LC'][i] = locflag
|
|
3404
3197
|
else:
|
|
@@ -3410,31 +3203,21 @@ class DsArch(PgArch, PgMeta):
|
|
|
3410
3203
|
omove = 0
|
|
3411
3204
|
dslocflags.add('G')
|
|
3412
3205
|
if wmove:
|
|
3413
|
-
|
|
3414
|
-
|
|
3415
|
-
sfrom = 0
|
|
3416
|
-
else:
|
|
3417
|
-
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3206
|
+
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
3207
|
+
sfrom = 0
|
|
3418
3208
|
if not stat:
|
|
3419
3209
|
self.RETSTAT = 1
|
|
3420
3210
|
continue
|
|
3421
3211
|
wcnt += 1
|
|
3422
3212
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3423
3213
|
if omove:
|
|
3424
|
-
|
|
3425
|
-
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3426
|
-
elif wmove:
|
|
3427
|
-
stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3428
|
-
else:
|
|
3429
|
-
stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3430
|
-
ofrom = 0
|
|
3214
|
+
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
3431
3215
|
if not stat:
|
|
3432
3216
|
self.RETSTAT = 1
|
|
3433
3217
|
continue
|
|
3434
3218
|
ocnt += 1
|
|
3435
3219
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3436
3220
|
if sfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
|
|
3437
|
-
if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
|
|
3438
3221
|
if pgrec['bid'] and not self.params['QF'][i]: self.params['QF'][i] = pgrec['bid']
|
|
3439
3222
|
if pgrec['gindex'] and not self.params['GI'][i]: self.params['GI'][i] = pgrec['gindex']
|
|
3440
3223
|
if pgrec['data_size'] and not self.params['SZ'][i]: self.params['SZ'][i] = pgrec['data_size']
|
|
@@ -3548,7 +3331,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
3548
3331
|
wcnd = "wid = {}".format(pgrec['wid'])
|
|
3549
3332
|
if (oflag + wflag) > 0:
|
|
3550
3333
|
locflag = "O" if oflag else "G"
|
|
3551
|
-
lrec = {'locflag'
|
|
3334
|
+
lrec = {'locflag': locflag}
|
|
3552
3335
|
mcnt += self.pgupdt_wfile(dsid, lrec, wcnd, self.LGEREX)
|
|
3553
3336
|
else:
|
|
3554
3337
|
ccnt = self.record_filenumber(dsid, pgrec['gindex'], 4, (pgrec['type'] if pgrec['status'] == 'P' else ''), -1, -pgrec['data_size'])
|
|
@@ -3682,13 +3465,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3682
3465
|
if tmpgs: self.params['GI'] = self.params['OG']
|
|
3683
3466
|
aolds = [None]*self.ALLCNT
|
|
3684
3467
|
solds = [None]*self.ALLCNT
|
|
3685
|
-
oolds = [None]*self.ALLCNT
|
|
3686
3468
|
tolds = [None]*self.ALLCNT
|
|
3687
3469
|
for i in range(self.ALLCNT):
|
|
3688
3470
|
type = self.params['OT'][i] if 'OT' in self.params and self.params['OT'][i] else 'P'
|
|
3689
3471
|
aolds[i] = self.get_saved_path(i, self.params['RF'][i], 5, type)
|
|
3690
3472
|
solds[i] = self.get_saved_path(i, self.params['RF'][i], 4, type)
|
|
3691
|
-
oolds[i] = self.get_object_path(solds[i], self.params['DS'])
|
|
3692
3473
|
tolds[i] = type
|
|
3693
3474
|
if tmpds: self.params['DS'] = tmpds
|
|
3694
3475
|
if tmpgs: self.params['GI'] = tmpgs
|
|
@@ -3696,13 +3477,11 @@ class DsArch(PgArch, PgMeta):
|
|
|
3696
3477
|
init = 1 if (tmpds or tmpgs) else 0
|
|
3697
3478
|
anews = [None]*self.ALLCNT
|
|
3698
3479
|
snews = [None]*self.ALLCNT
|
|
3699
|
-
onews = [None]*self.ALLCNT
|
|
3700
3480
|
tnews = [None]*self.ALLCNT
|
|
3701
3481
|
for i in range(self.ALLCNT):
|
|
3702
3482
|
type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'P'
|
|
3703
3483
|
anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
|
|
3704
3484
|
snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
|
|
3705
|
-
onews[i] = self.get_object_path(snews[i], dsid)
|
|
3706
3485
|
tnews[i] = type
|
|
3707
3486
|
init = 0
|
|
3708
3487
|
fnames = "FIT"
|
|
@@ -3714,13 +3493,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
3714
3493
|
if bidx == self.ALLCNT: return
|
|
3715
3494
|
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
3716
3495
|
chksize = self.PGLOG['DSCHECK']['size']
|
|
3717
|
-
reorder = self.MODCNT = scnt =
|
|
3496
|
+
reorder = self.MODCNT = scnt = 0
|
|
3718
3497
|
for i in range(bidx, self.ALLCNT):
|
|
3719
3498
|
if i > bidx and ((i-bidx)%20) == 0:
|
|
3720
3499
|
if self.PGLOG['DSCHECK']:
|
|
3721
3500
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
3722
3501
|
if 'EM' in self.params:
|
|
3723
|
-
self.PGLOG['PRGMSG'] = "{}/{}/{}
|
|
3502
|
+
self.PGLOG['PRGMSG'] = "{}/{}/{}, Disk/Record/Proccessed, of {} Saved file{} moved".format(scnt, self.MODCNT, i, self.ALLCNT, s)
|
|
3724
3503
|
type = tolds[i]
|
|
3725
3504
|
pgrec = self.pgget(tname, "*", "sfile = '{}' AND dsid = '{}' AND type = '{}'".format(solds[i], self.params['OD'], type), self.LGEREX)
|
|
3726
3505
|
if not pgrec:
|
|
@@ -3745,27 +3524,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
3745
3524
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3746
3525
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3747
3526
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3748
|
-
|
|
3749
|
-
smove = 1
|
|
3750
|
-
# if pgrec['locflag'] == 'O':
|
|
3751
|
-
# smove = 0
|
|
3752
|
-
# elif pgrec['locflag'] == 'G':
|
|
3753
|
-
# omove = 0
|
|
3754
|
-
if smove and aolds[i] != anews[i]:
|
|
3527
|
+
if aolds[i] != anews[i]:
|
|
3755
3528
|
if not self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE):
|
|
3756
3529
|
self.RETSTAT = 1
|
|
3757
3530
|
continue
|
|
3758
3531
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3759
3532
|
scnt += 1
|
|
3760
|
-
if omove and oolds[i] != onews[i]:
|
|
3761
|
-
if not self.move_object_file(onews[i], oolds[i], bucket, bucket, self.PGOPT['emerol']|self.OVERRIDE):
|
|
3762
|
-
self.RETSTAT = 1
|
|
3763
|
-
continue
|
|
3764
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
3765
|
-
ocnt += 1
|
|
3766
3533
|
self.set_one_savedfile(i, pgrec, snews[i], fnames, tnews[i], None, tmpds)
|
|
3767
3534
|
if pgrec['bid']: self.save_move_info(pgrec['bid'], solds[i], type, 'S', self.params['OD'], snews[i], tnews[i], 'S', dsid)
|
|
3768
|
-
self.pglog("{}/{}
|
|
3535
|
+
self.pglog("{}/{}, Disk/Record, of {} Saved file{} moved".format(scnt, self.MODCNT, self.ALLCNT, s), self.PGOPT['emllog'])
|
|
3769
3536
|
if self.PGLOG['DSCHECK']:
|
|
3770
3537
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
3771
3538
|
if 'ON' in self.params:
|
|
@@ -3880,7 +3647,6 @@ class DsArch(PgArch, PgMeta):
|
|
|
3880
3647
|
dsid = self.params['DS']
|
|
3881
3648
|
dcnd = "dsid = '{}'".format(dsid)
|
|
3882
3649
|
frombucket = self.PGLOG['OBJCTBKT']
|
|
3883
|
-
tobucket = "gdex-decsdata"
|
|
3884
3650
|
bidx = chksize = 0
|
|
3885
3651
|
tmpds = tmpgs = None
|
|
3886
3652
|
if 'OD' not in self.params:
|
|
@@ -3921,20 +3687,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
3921
3687
|
init = 1 if (tmpds or tmpgs) else 0
|
|
3922
3688
|
anews = [None]*self.ALLCNT
|
|
3923
3689
|
snews = [None]*self.ALLCNT
|
|
3924
|
-
onews = [None]*self.ALLCNT
|
|
3925
3690
|
tnews = [None]*self.ALLCNT
|
|
3926
3691
|
for i in range(self.ALLCNT):
|
|
3927
3692
|
type = self.params['ST'][i] if 'ST' in self.params and self.params['ST'][i] else 'V'
|
|
3928
3693
|
anews[i] = self.get_saved_path(i, self.params['SF'][i], 5, type, init)
|
|
3929
3694
|
snews[i] = self.get_saved_path(i, self.params['SF'][i], 4, type)
|
|
3930
|
-
onews[i] = self.join_paths(dsid, snews[i])
|
|
3931
3695
|
tnews[i] = type
|
|
3932
3696
|
init = 0
|
|
3933
3697
|
self.validate_multiple_values(tname, self.ALLCNT)
|
|
3934
3698
|
if self.PGLOG['DSCHECK']:
|
|
3935
3699
|
bidx = self.set_dscheck_fcount(self.ALLCNT, self.PGOPT['extlog'])
|
|
3936
3700
|
if bidx > 0:
|
|
3937
|
-
self.pglog("bidx of self.ALLCNT Web to Saved files processed for move", self.PGOPT['emllog'])
|
|
3701
|
+
self.pglog(f"{bidx} of {self.ALLCNT} Web to Saved files processed for move", self.PGOPT['emllog'])
|
|
3938
3702
|
if bidx == self.ALLCNT: return
|
|
3939
3703
|
self.set_rearchive_filenumber(dsid, bidx, self.ALLCNT, 4)
|
|
3940
3704
|
chksize = self.PGLOG['DSCHECK']['size']
|
|
@@ -3961,7 +3725,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
3961
3725
|
if self.PGLOG['DSCHECK']:
|
|
3962
3726
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
3963
3727
|
if 'EM' in self.params:
|
|
3964
|
-
self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {}
|
|
3728
|
+
self.PGLOG['PRGMSG'] = "{}/{}/{}/{}/{}, Disk/Object/RecordDeleted/RecordAdded/Proccessed, of {} Web file{} moved".format(wcnt, ocnt, dcnt, self.ADDCNT, i, self.ALLCNT, s)
|
|
3965
3729
|
type = tolds[i]
|
|
3966
3730
|
pgrec = self.pgget_wfile(self.params['OD'], "*", "wfile = '{}' AND type = '{}'".format(wolds[i], type), self.LGEREX)
|
|
3967
3731
|
if not pgrec:
|
|
@@ -3980,45 +3744,23 @@ class DsArch(PgArch, PgMeta):
|
|
|
3980
3744
|
not self.pgget("dsgroup", "", "{} and gindex = {}".format(dcnd, pgrec['gindex']), self.PGOPT['extlog'])):
|
|
3981
3745
|
self.pglog("Group Index {} is not in RDADB for {}\n".format(pgrec['gindex'], dsid) +
|
|
3982
3746
|
"Specify Original/New group index via options -OG/-GI", self.PGOPT['extlog'])
|
|
3983
|
-
ofrom = wfrom =
|
|
3984
|
-
omove = 0
|
|
3747
|
+
ofrom = wfrom = 1
|
|
3985
3748
|
locflag = pgrec['locflag']
|
|
3986
3749
|
if locflag == 'O':
|
|
3987
3750
|
wfrom = 0
|
|
3988
3751
|
elif locflag == 'G':
|
|
3989
3752
|
ofrom = 0
|
|
3990
|
-
|
|
3991
|
-
|
|
3992
|
-
|
|
3993
|
-
# locflag = self.params['LC'][i]
|
|
3994
|
-
# if locflag == 'O':
|
|
3995
|
-
# smove = 0
|
|
3996
|
-
# elif locflag == 'G':
|
|
3997
|
-
# omove = 0
|
|
3998
|
-
if smove:
|
|
3999
|
-
if wfrom:
|
|
4000
|
-
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
4001
|
-
wfrom = 0
|
|
4002
|
-
else:
|
|
4003
|
-
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4004
|
-
if not stat:
|
|
4005
|
-
self.RETSTAT = 1
|
|
4006
|
-
continue
|
|
3753
|
+
if wfrom:
|
|
3754
|
+
stat = self.move_local_file(anews[i], aolds[i], self.PGOPT['emerol']|self.OVERRIDE)
|
|
3755
|
+
wfrom = 0
|
|
4007
3756
|
wcnt += 1
|
|
4008
|
-
|
|
4009
|
-
|
|
4010
|
-
if wfrom:
|
|
4011
|
-
stat = self.local_copy_object(onews[i], aolds[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4012
|
-
elif smove:
|
|
4013
|
-
stat = self.local_copy_object(onews[i], anews[i], tobucket, None, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4014
|
-
else:
|
|
4015
|
-
stat = self.move_object_file(onews[i], oolds[i], tobucket, frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4016
|
-
ofrom = 0
|
|
4017
|
-
if not stat:
|
|
4018
|
-
self.RETSTAT = 1
|
|
4019
|
-
continue
|
|
3757
|
+
else:
|
|
3758
|
+
stat = self.object_copy_local(anews[i], oolds[i], frombucket, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4020
3759
|
ocnt += 1
|
|
4021
|
-
|
|
3760
|
+
if not stat:
|
|
3761
|
+
self.RETSTAT = 1
|
|
3762
|
+
continue
|
|
3763
|
+
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4022
3764
|
if wfrom: self.delete_local_file(aolds[i], self.PGOPT['emerol'])
|
|
4023
3765
|
if ofrom: self.delete_object_file(oolds[i], frombucket, self.PGOPT['emerol'])
|
|
4024
3766
|
if self.PGOPT['GXTYP'].find(type) > -1 and ('DX' in self.params or pgrec and pgrec['meta_link'] and pgrec['meta_link'] != 'N' and 'KM' not in self.params):
|
|
@@ -4052,7 +3794,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4052
3794
|
|
|
4053
3795
|
# get date/time/size info from given file record
|
|
4054
3796
|
def get_file_origin_info(self, fname, pgrec):
|
|
4055
|
-
info = {'isfile'
|
|
3797
|
+
info = {'isfile': (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size': pgrec['data_size']}
|
|
4056
3798
|
info['fname'] = op.basename(fname)
|
|
4057
3799
|
info['date_modified'] = pgrec['date_modified']
|
|
4058
3800
|
info['time_modified'] = pgrec['time_modified']
|
|
@@ -4065,10 +3807,9 @@ class DsArch(PgArch, PgMeta):
|
|
|
4065
3807
|
tname = 'sfile'
|
|
4066
3808
|
dsid = self.params['DS']
|
|
4067
3809
|
dcnd = "dsid = '{}'".format(dsid)
|
|
4068
|
-
bucket = "gdex-decsdata"
|
|
4069
3810
|
s = 's' if self.ALLCNT > 1 else ''
|
|
4070
3811
|
self.pglog("Delete {} Saved file{} from {} ...".format(self.ALLCNT, s, dsid), self.WARNLG)
|
|
4071
|
-
bidx = chksize = reorder = scnt =
|
|
3812
|
+
bidx = chksize = reorder = scnt = dcnt = 0
|
|
4072
3813
|
self.cache_group_info(self.ALLCNT, 0)
|
|
4073
3814
|
self.validate_multiple_options(self.ALLCNT, ["ST", 'VI', 'QF', 'LC'])
|
|
4074
3815
|
if self.PGLOG['DSCHECK']:
|
|
@@ -4083,7 +3824,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4083
3824
|
if self.PGLOG['DSCHECK']:
|
|
4084
3825
|
self.set_dscheck_dcount(i, chksize, self.PGOPT['extlog'])
|
|
4085
3826
|
if 'EM' in self.params:
|
|
4086
|
-
self.PGLOG['PRGMSG'] = "{}/{}/
|
|
3827
|
+
self.PGLOG['PRGMSG'] = "{}/{}/i, Disk/Record/processed, of {} Saved file{} deleted".format(scnt, dcnt, i, self.ALLCNT, s)
|
|
4087
3828
|
sfile = self.params['SF'][i]
|
|
4088
3829
|
if 'ST' in self.params and self.params['ST'][i]:
|
|
4089
3830
|
type = self.params['ST'][i]
|
|
@@ -4095,24 +3836,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
4095
3836
|
if not pgrec:
|
|
4096
3837
|
self.pglog("{}-{}: Type '{}' Saved file is not in RDADB".format(dsid, sfile, type), self.PGOPT['errlog'])
|
|
4097
3838
|
continue
|
|
4098
|
-
sdel =
|
|
4099
|
-
|
|
4100
|
-
locflag
|
|
4101
|
-
# locflag = pgrec['locflag']
|
|
4102
|
-
# if locflag == 'O':
|
|
4103
|
-
# sflag = 0
|
|
4104
|
-
# elif locflag == 'G':
|
|
4105
|
-
# oflag = 0
|
|
4106
|
-
# elif locflag == 'C':
|
|
4107
|
-
# sflag = oflag = 0
|
|
3839
|
+
sdel = sflag = 1
|
|
3840
|
+
locflag = pgrec['locflag']
|
|
3841
|
+
if locflag != 'G': sflag = 0
|
|
4108
3842
|
if 'LC' in self.params and self.params['LC'][i]: locflag = self.params['LC'][i]
|
|
4109
|
-
if locflag
|
|
4110
|
-
|
|
4111
|
-
elif locflag == 'G':
|
|
4112
|
-
odel = 0
|
|
4113
|
-
elif locflag == 'C':
|
|
4114
|
-
sdel = odel = 0
|
|
4115
|
-
if (sflag+oflag) == (sdel+odel):
|
|
3843
|
+
if locflag != 'G': sdel = 0
|
|
3844
|
+
if sflag == sdel:
|
|
4116
3845
|
vindex = self.params['VI'][i] if 'VI' in self.params else pgrec['vindex']
|
|
4117
3846
|
if vindex:
|
|
4118
3847
|
self.pglog(sfile + ": Saved file is version controlled, add option -vi 0 to force delete", self.PGOPT['errlog'])
|
|
@@ -4129,24 +3858,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
4129
3858
|
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4130
3859
|
elif self.check_local_file(afile) is None:
|
|
4131
3860
|
sflag = 0
|
|
4132
|
-
if
|
|
4133
|
-
|
|
4134
|
-
if self.delete_object_file(ofile, bucket, self.PGOPT['emerol']):
|
|
4135
|
-
ocnt += 1
|
|
4136
|
-
oflag = 0
|
|
4137
|
-
if self.PGLOG['DSCHECK']: chksize += pgrec['data_size']
|
|
4138
|
-
elif self.check_object_file(ofile, bucket) is None:
|
|
4139
|
-
oflag = 0
|
|
4140
|
-
if (oflag + sflag) > 0:
|
|
4141
|
-
locflag = "O" if oflag else "G"
|
|
4142
|
-
self.pgexec("UPDATE sfile SET locflag = '{}' WHERE sid = {}".format(locflag, pgrec['sid']), self.LGEREX)
|
|
3861
|
+
if sflag > 0:
|
|
3862
|
+
self.pgexec("UPDATE sfile SET locflag = 'G' WHERE sid = {}".format(pgrec['sid']), self.LGEREX)
|
|
4143
3863
|
else:
|
|
4144
3864
|
ccnt = self.record_filenumber(dsid, pgrec['gindex'], 8, 'P', -1, -pgrec['data_size'])
|
|
4145
3865
|
fcnt = self.pgdel_sfile("sid = {}".format(pgrec['sid']), self.LGEREX)
|
|
4146
3866
|
if fcnt: dcnt += fcnt
|
|
4147
3867
|
if ccnt: self.save_filenumber(dsid, 8, 1, fcnt)
|
|
4148
|
-
if (scnt +
|
|
4149
|
-
self.pglog("{}/{}
|
|
3868
|
+
if (scnt + dcnt) > 0:
|
|
3869
|
+
self.pglog("{}/{}, Disk/Record, of {} Saved file{} deleted for {}".format(scnt, dcnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
4150
3870
|
if self.PGLOG['DSCHECK']:
|
|
4151
3871
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
4152
3872
|
if 'ON' in self.params:
|
|
@@ -4160,7 +3880,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4160
3880
|
tname = 'bfile'
|
|
4161
3881
|
dsid = self.params['DS']
|
|
4162
3882
|
dcnd = "dsid = '{}'".format(dsid)
|
|
4163
|
-
brec = {'bid'
|
|
3883
|
+
brec = {'bid': 0}
|
|
4164
3884
|
bkend = "gdex-quasar"
|
|
4165
3885
|
drend = "gdex-quasar-drdata"
|
|
4166
3886
|
s = 's' if self.ALLCNT > 1 else ''
|
|
@@ -4243,7 +3963,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4243
3963
|
metacnt = prdcnt = savedcnt = webcnt = rccnt = pgcnt = modcnt = 0
|
|
4244
3964
|
twcnt = tscnt = 0
|
|
4245
3965
|
for i in range(self.ALLCNT):
|
|
4246
|
-
record = {'gindex'
|
|
3966
|
+
record = {'gindex': self.params['GI'][i]}
|
|
4247
3967
|
gcnd = "gindex = {}".format(self.params['OG'][i])
|
|
4248
3968
|
cnd = "{} AND {}".format(dcnd, gcnd)
|
|
4249
3969
|
modcnt += self.pgupdt(tname, record, cnd, self.LGEREX)
|
|
@@ -4256,12 +3976,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
4256
3976
|
savedcnt += self.pgupdt("sfile", record, cnd, self.LGEREX)
|
|
4257
3977
|
rccnt += self.pgupdt("rcrqst", record, cnd, self.LGEREX)
|
|
4258
3978
|
if tgroups[i]:
|
|
4259
|
-
tgrec = {'tindex'
|
|
3979
|
+
tgrec = {'tindex': self.params['GI'][i]}
|
|
4260
3980
|
tcnd = "tindex = {}".format(self.params['OG'][i])
|
|
4261
3981
|
cnd = "{} AND {}".format(dcnd, tcnd)
|
|
4262
3982
|
twcnt += self.pgupdt_wfile(dsid, tgrec, tcnd, self.LGEREX)
|
|
4263
3983
|
tscnt += self.pgupdt("sfile", tgrec, cnd, self.LGEREX)
|
|
4264
|
-
pgrec = {'pindex'
|
|
3984
|
+
pgrec = {'pindex': self.params['GI'][i]}
|
|
4265
3985
|
cnd = "pindex = {} AND {}".format(self.params['OG'][i], dcnd)
|
|
4266
3986
|
pgcnt += self.pgupdt(tname, pgrec, cnd, self.LGEREX)
|
|
4267
3987
|
self.pglog("{} of {} group{} changed".format(modcnt, self.ALLCNT, s), self.LOGWRN)
|
|
@@ -4295,7 +4015,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4295
4015
|
cnt = 0
|
|
4296
4016
|
getkeys = 1
|
|
4297
4017
|
if cnt > 0:
|
|
4298
|
-
values = {'okeys'
|
|
4018
|
+
values = {'okeys': [], 'value': []}
|
|
4299
4019
|
for i in range(cnt):
|
|
4300
4020
|
pgrec = self.pgget("dsokeys", "value", "{}AND okey = '{}'".fomrat(cond, kvalues[i]), self.LGWNEX)
|
|
4301
4021
|
if pgrec:
|
|
@@ -4342,7 +4062,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4342
4062
|
pgrec['value'] = value
|
|
4343
4063
|
mcnt += self.pgupdt("dsokeys", pgrec, cond, self.LGWNEX)
|
|
4344
4064
|
else:
|
|
4345
|
-
pgrec = {'dsid'
|
|
4065
|
+
pgrec = {'dsid': dsid, 'okey': key, 'value': value}
|
|
4346
4066
|
acnt += self.pgadd("dsokeys", pgrec, self.LGWNEX)
|
|
4347
4067
|
self.pglog("{}/{}/{} of {} key/value pairs added/modified/deleted for {}!".format(acnt, mcnt, dcnt, cnt, dsid), self.LOGWRN)
|
|
4348
4068
|
return (acnt + mcnt + dcnt)
|
|
@@ -4352,8 +4072,8 @@ class DsArch(PgArch, PgMeta):
|
|
|
4352
4072
|
date = self.curdate()
|
|
4353
4073
|
cond = "wid = {} and date = '{}'".format(pgrec['wid'], date)
|
|
4354
4074
|
if not self.pgget("wmove", "", cond, self.LGWNEX):
|
|
4355
|
-
record = {'dsid'
|
|
4356
|
-
'wfile'
|
|
4075
|
+
record = {'dsid': pgrec['dsid'], 'uid': self.PGOPT['UID'],
|
|
4076
|
+
'wfile': pgrec['wfile'], 'wid': pgrec['wid'], 'date': date}
|
|
4357
4077
|
self.pgadd("wmove", record, self.LGWNEX)
|
|
4358
4078
|
|
|
4359
4079
|
# reset file counts for saved groups
|
|
@@ -4588,7 +4308,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4588
4308
|
addcnt += 1
|
|
4589
4309
|
if not doi: vidx = 0
|
|
4590
4310
|
if vidx:
|
|
4591
|
-
vrec = {'vindex'
|
|
4311
|
+
vrec = {'vindex': vidx}
|
|
4592
4312
|
vcnd = "type = 'D' AND vindex = 0"
|
|
4593
4313
|
fcnt = self.pgupdt_wfile(dsid, vrec, vcnd, self.PGOPT['extlog'])
|
|
4594
4314
|
if fcnt > 0:
|
|
@@ -4640,7 +4360,7 @@ class DsArch(PgArch, PgMeta):
|
|
|
4640
4360
|
self.pglog("{}: Cannot terminate for DOI {} is asscoated to Version control {}".format(vinfo, pgrec['doi'], orec['vindex']), self.LOGERR)
|
|
4641
4361
|
continue
|
|
4642
4362
|
doicnt += 1
|
|
4643
|
-
record = {'status'
|
|
4363
|
+
record = {'status': "H"}
|
|
4644
4364
|
record['end_date'] = self.params['ED'][i] if 'ED' in self.params else self.curdate()
|
|
4645
4365
|
record['end_time'] = self.params['ET'][i] if 'ET' in self.params else self.curtime()
|
|
4646
4366
|
if self.pgupdt("dsvrsn", record, cnd, self.PGOPT['extlog']):
|
rda_python_dsarch/pg_arch.py
CHANGED
|
@@ -2,13 +2,13 @@ rda_python_dsarch/PgArch.py,sha256=d1Y87MT49tgQygLAffNff9Qs4c12KXspYyDG8liSnpQ,5
|
|
|
2
2
|
rda_python_dsarch/PgMeta.py,sha256=FV4GytgnJK00FUlyM6iHqpNd7-XTVHO1j5tZtcLTWgY,26845
|
|
3
3
|
rda_python_dsarch/__init__.py,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1
|
|
4
4
|
rda_python_dsarch/ds_arch.py,sha256=6bR0y6_rhIXyvbuYd-oZfL1YcaHhWBkojjzYz1yCrVg,233810
|
|
5
|
-
rda_python_dsarch/dsarch.py,sha256=
|
|
5
|
+
rda_python_dsarch/dsarch.py,sha256=P68EY1CBJJ60ueH7rxqH-fnUpRNgU9HS4Fc_wUAStBA,232970
|
|
6
6
|
rda_python_dsarch/dsarch.usg,sha256=vsxGneGCIyzqZx5mwQ8xkTmdNmuny56LgokgDkWt5xE,132034
|
|
7
|
-
rda_python_dsarch/pg_arch.py,sha256=
|
|
7
|
+
rda_python_dsarch/pg_arch.py,sha256=hMAq6IwWzDjjiSkouVcPNZzNJ0nmTT88y-GZRi8SJhE,64599
|
|
8
8
|
rda_python_dsarch/pg_meta.py,sha256=f_7w9UxhJk1i-QfkmDvNCPrrDYGcOkNPNImXLNZw0sA,29548
|
|
9
|
-
rda_python_dsarch-2.0.
|
|
10
|
-
rda_python_dsarch-2.0.
|
|
11
|
-
rda_python_dsarch-2.0.
|
|
12
|
-
rda_python_dsarch-2.0.
|
|
13
|
-
rda_python_dsarch-2.0.
|
|
14
|
-
rda_python_dsarch-2.0.
|
|
9
|
+
rda_python_dsarch-2.0.4.dist-info/licenses/LICENSE,sha256=1dck4EAQwv8QweDWCXDx-4Or0S8YwiCstaso_H57Pno,1097
|
|
10
|
+
rda_python_dsarch-2.0.4.dist-info/METADATA,sha256=cvBOpD-O0Bjzf-oeGXw4XbhXI0l4Zy-kFnUrfFTre_U,670
|
|
11
|
+
rda_python_dsarch-2.0.4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
12
|
+
rda_python_dsarch-2.0.4.dist-info/entry_points.txt,sha256=GR7ShgBv8qixS2l4ufAu-jw3N09pbo9zRewwPHqaOxo,60
|
|
13
|
+
rda_python_dsarch-2.0.4.dist-info/top_level.txt,sha256=-vwcMgUXbBdHscvAKB3De9zKac3m_s_FaKIdApP1FGY,18
|
|
14
|
+
rda_python_dsarch-2.0.4.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|