rda-python-dsquasar 1.0.9__tar.gz → 1.0.10__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/PKG-INFO +1 -1
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/pyproject.toml +1 -1
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar/dsquasar.py +20 -20
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/PKG-INFO +1 -1
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/LICENSE +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/README.md +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/setup.cfg +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar/__init__.py +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/SOURCES.txt +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/dependency_links.txt +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/entry_points.txt +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/requires.txt +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/top_level.txt +0 -0
- {rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/tests/test_dsquasar.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_dsquasar
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.10
|
|
4
4
|
Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar
|
{rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar/dsquasar.py
RENAMED
|
@@ -10,7 +10,7 @@
|
|
|
10
10
|
# Purpose : check Backup flags set in dataset.backflag and/or dsgroup.backflag of RDADB
|
|
11
11
|
# to gather Web and Saved bfiles, tar them into larger (> 2GB) bfiles, and
|
|
12
12
|
# back them up onto Quasar Backup Servers, at the Globus end points of
|
|
13
|
-
# NCAR
|
|
13
|
+
# NCAR GDEX Quasar and/or NCAR GDEX Quasar Drdata, for Backup and/or
|
|
14
14
|
# desaster recovery copies, respectively. The file tarring and backing up
|
|
15
15
|
# processes are completed in utility program dsarch.
|
|
16
16
|
#
|
|
@@ -91,7 +91,7 @@ TFCOUNT = 100 # if file count is greater, use MINSIZE for tar file
|
|
|
91
91
|
SUBLMTS = 2000 # file count limit for a sub-group
|
|
92
92
|
|
|
93
93
|
PGBACK = {
|
|
94
|
-
'workdir' : "{}/{}/quasar_backup".format(PgLOG.PGLOG['UPDTWKP'], PgLOG.PGLOG['
|
|
94
|
+
'workdir' : "{}/{}/quasar_backup".format(PgLOG.PGLOG['UPDTWKP'], PgLOG.PGLOG['GDEXUSER']),
|
|
95
95
|
'mproc' : 1,
|
|
96
96
|
'action' : CTACTS,
|
|
97
97
|
'chgdays' : 0,
|
|
@@ -266,7 +266,7 @@ def dump_statistics_action(bopts, dsids):
|
|
|
266
266
|
dump_dataset_files(dsfiles, 'D')
|
|
267
267
|
if PGBACK['chgdays']: return # no further checking
|
|
268
268
|
|
|
269
|
-
# changed
|
|
269
|
+
# changed GDEX files ready to be backed up
|
|
270
270
|
PGBACK['chgdays'] = 1
|
|
271
271
|
dsfiles = {'B' : {}, 'D' : {}}
|
|
272
272
|
gather_dataset_files(dsfiles, dsids, False)
|
|
@@ -341,7 +341,7 @@ def add_checksum_action(bopts, dsids, dstart):
|
|
|
341
341
|
process_dataset_mcsfiles(dsfiles, 'D')
|
|
342
342
|
|
|
343
343
|
#
|
|
344
|
-
# check available
|
|
344
|
+
# check available GDEX files to create input files
|
|
345
345
|
#
|
|
346
346
|
def create_infile_action(bopts, dsids, dstart):
|
|
347
347
|
|
|
@@ -515,7 +515,7 @@ def backup_dataset_files(dsfiles, backflag):
|
|
|
515
515
|
dcnt = len(qinfo['qdsids'])
|
|
516
516
|
ssize = PgUtil.format_float_value(qinfo['qsize'])
|
|
517
517
|
dmsg = qinfo['qdsids'][0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
518
|
-
fmsg = "file{} for {}({})
|
|
518
|
+
fmsg = "file{} for {}({}) GDEX {}files of {}".format(s, fcnt, ssize, cmsg, dmsg)
|
|
519
519
|
msg = "{}: {} {} {}".format(amsg, qcnt, bmsg, fmsg)
|
|
520
520
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
521
521
|
|
|
@@ -527,7 +527,7 @@ def backup_dataset_files(dsfiles, backflag):
|
|
|
527
527
|
dsids = qinfo['dsids']
|
|
528
528
|
dcnt = len(dsids)
|
|
529
529
|
dmsg = dsids[0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
530
|
-
msg = "{}: {}({})
|
|
530
|
+
msg = "{}: {}({}) GDEX {}file{} of {} for next {}".format(amsg, fcnt, ssize, cmsg, s, dmsg, bmsg)
|
|
531
531
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
532
532
|
|
|
533
533
|
#
|
|
@@ -584,7 +584,7 @@ def backup_dataset_infiles(dsfiles, backflag):
|
|
|
584
584
|
dcnt = len(qinfo['qdsids'])
|
|
585
585
|
ssize = PgUtil.format_float_value(qinfo['qsize'])
|
|
586
586
|
dmsg = qinfo['qdsids'][0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
587
|
-
fmsg = "file{} for {}({})
|
|
587
|
+
fmsg = "file{} for {}({}) GDEX files of {}".format(s, fcnt, ssize, dmsg)
|
|
588
588
|
msg = "{}: {} {} {}".format(amsg, qcnt, bmsg, fmsg)
|
|
589
589
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
590
590
|
|
|
@@ -629,7 +629,7 @@ def backup_dataset_tarfiles(dsfiles, backflag):
|
|
|
629
629
|
dcnt = len(qinfo['qdsids'])
|
|
630
630
|
ssize = PgUtil.format_float_value(qinfo['qsize'])
|
|
631
631
|
dmsg = qinfo['qdsids'][0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
632
|
-
fmsg = "file{} for {}({})
|
|
632
|
+
fmsg = "file{} for {}({}) GDEX files of {}".format(s, fcnt, ssize, dmsg)
|
|
633
633
|
msg = "{}: {} {} {}".format(amsg, qcnt, bmsg, fmsg)
|
|
634
634
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
635
635
|
|
|
@@ -1189,7 +1189,7 @@ def gather_dataset_files(dsfiles, dsids, unlock = True):
|
|
|
1189
1189
|
s = 's' if fcnt > 1 else ''
|
|
1190
1190
|
bmsg = BACKMSG[PGBACK['backflag']] if PGBACK['backflag'] else 'backup'
|
|
1191
1191
|
cmsg = 'Changed ' if PGBACK['chgdays'] else ''
|
|
1192
|
-
PgLOG.pglog("{}
|
|
1192
|
+
PgLOG.pglog("{} GDEX {}file{} found ready to {}".format(fcnt, cmsg, s, bmsg), LOGACT)
|
|
1193
1193
|
|
|
1194
1194
|
return fcnt
|
|
1195
1195
|
|
|
@@ -2028,7 +2028,7 @@ def get_backup_mcsfile(dsfiles, pgrec):
|
|
|
2028
2028
|
|
|
2029
2029
|
|
|
2030
2030
|
#
|
|
2031
|
-
# dump statistics for
|
|
2031
|
+
# dump statistics for GDEX files available to create input files
|
|
2032
2032
|
#
|
|
2033
2033
|
def dump_dataset_files(dsfiles, backflag):
|
|
2034
2034
|
|
|
@@ -2072,7 +2072,7 @@ def dump_dataset_files(dsfiles, backflag):
|
|
|
2072
2072
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
2073
2073
|
|
|
2074
2074
|
#
|
|
2075
|
-
#
|
|
2075
|
+
# GDEX files included in input files
|
|
2076
2076
|
#
|
|
2077
2077
|
def dump_dataset_infiles(dsfiles, backflag):
|
|
2078
2078
|
|
|
@@ -2095,11 +2095,11 @@ def dump_dataset_infiles(dsfiles, backflag):
|
|
|
2095
2095
|
dcnt = len(dsids)
|
|
2096
2096
|
ssize = PgUtil.format_float_value(size)
|
|
2097
2097
|
dmsg = dsids[0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
2098
|
-
msg = "{}({}) Input files for {}
|
|
2098
|
+
msg = "{}({}) Input files for {} GDEX files of {} to be tarred for {}".format(bcnt, ssize, fcnt, dmsg, bmsg)
|
|
2099
2099
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
2100
2100
|
|
|
2101
2101
|
#
|
|
2102
|
-
# tarred
|
|
2102
|
+
# tarred GDEX files to backup
|
|
2103
2103
|
#
|
|
2104
2104
|
def dump_dataset_tarfiles(dsfiles, backflag):
|
|
2105
2105
|
|
|
@@ -2120,7 +2120,7 @@ def dump_dataset_tarfiles(dsfiles, backflag):
|
|
|
2120
2120
|
ssize = PgUtil.format_float_value(size)
|
|
2121
2121
|
bmsg = BACKMSG[backflag]
|
|
2122
2122
|
dmsg = dsids[0] if dcnt == 1 else "{} datasets".format(dcnt)
|
|
2123
|
-
msg = "{} ({}) Tarred files for {}
|
|
2123
|
+
msg = "{} ({}) Tarred files for {} GDEX files of {} for {}".format(bcnt, ssize, fcnt, dmsg, bmsg)
|
|
2124
2124
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
2125
2125
|
|
|
2126
2126
|
#
|
|
@@ -2153,7 +2153,7 @@ def dump_dataset_bckfiles(dsfiles, backflag):
|
|
|
2153
2153
|
ssize = PgUtil.format_float_value(qinfo['size'])
|
|
2154
2154
|
bmsg = BACKMSG[backflag]
|
|
2155
2155
|
dmsg = qinfo['dsids'][0] if dscnt == 1 else "{} datasets".format(dscnt)
|
|
2156
|
-
msg = "{} ({}) {} files for {}
|
|
2156
|
+
msg = "{} ({}) {} files for {} GDEX files of {}".format(bcnt, ssize, bmsg, qinfo['fcnt'], dmsg)
|
|
2157
2157
|
PgLOG.pglog(INDENT + msg, LOGACT)
|
|
2158
2158
|
|
|
2159
2159
|
indent = INDENT + INDENT
|
|
@@ -2163,7 +2163,7 @@ def dump_dataset_bckfiles(dsfiles, backflag):
|
|
|
2163
2163
|
msg = "{} {} files Missing Note fields of {}".format(bcnt, bmsg, dmsg)
|
|
2164
2164
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2165
2165
|
|
|
2166
|
-
msg = "{}
|
|
2166
|
+
msg = "{} GDEX files Changed after {}".format(pcnt, bmsg)
|
|
2167
2167
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2168
2168
|
if pcnt == 0: return
|
|
2169
2169
|
|
|
@@ -2171,28 +2171,28 @@ def dump_dataset_bckfiles(dsfiles, backflag):
|
|
|
2171
2171
|
dscnt = len(qinfo['cdsids'])
|
|
2172
2172
|
dmsg = qinfo['cdsids'][0] if dscnt == 1 else "{} datasets".format(dscnt)
|
|
2173
2173
|
ssize = PgUtil.format_float_value(qinfo['csize'])
|
|
2174
|
-
msg = "{} ({})
|
|
2174
|
+
msg = "{} ({}) GDEX file sizes Changed for {}".format(qinfo['ccnt'], ssize, dmsg)
|
|
2175
2175
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2176
2176
|
|
|
2177
2177
|
if qinfo['ucnt'] > 0:
|
|
2178
2178
|
dscnt = len(qinfo['udsids'])
|
|
2179
2179
|
dmsg = qinfo['udsids'][0] if dscnt == 1 else "{} datasets".format(dscnt)
|
|
2180
2180
|
ssize = PgUtil.format_float_value(qinfo['usize'])
|
|
2181
|
-
msg = "{} ({})
|
|
2181
|
+
msg = "{} ({}) GDEX files Updated & Re-done {} for {}".format(qinfo['ucnt'], ssize, bmsg, dmsg)
|
|
2182
2182
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2183
2183
|
|
|
2184
2184
|
if qinfo['dcnt'] > 0:
|
|
2185
2185
|
dscnt = len(qinfo['ddsids'])
|
|
2186
2186
|
dmsg = qinfo['ddsids'][0] if dscnt == 1 else "{} datasets".format(dscnt)
|
|
2187
2187
|
ssize = PgUtil.format_float_value(qinfo['dsize'])
|
|
2188
|
-
msg = "{} ({})
|
|
2188
|
+
msg = "{} ({}) GDEX files Got Deleted for {}".format(qinfo['dcnt'], ssize, dmsg)
|
|
2189
2189
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2190
2190
|
|
|
2191
2191
|
if qinfo['mcnt'] > 0:
|
|
2192
2192
|
dscnt = len(qinfo['mdsids'])
|
|
2193
2193
|
dmsg = qinfo['mdsids'][0] if dscnt == 1 else "{} datasets".format(dscnt)
|
|
2194
2194
|
ssize = PgUtil.format_float_value(qinfo['msize'])
|
|
2195
|
-
msg = "{} ({})
|
|
2195
|
+
msg = "{} ({}) GDEX files Moved for {}".format(qinfo['mcnt'], ssize, dmsg)
|
|
2196
2196
|
PgLOG.pglog(indent + msg, LOGACT)
|
|
2197
2197
|
|
|
2198
2198
|
#
|
{rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar.egg-info/PKG-INFO
RENAMED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: rda_python_dsquasar
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.10
|
|
4
4
|
Summary: RDA Python package to backup and recover RDA data archives to and from GLOBUS Quasar backup server
|
|
5
5
|
Author-email: Zaihua Ji <zji@ucar.edu>
|
|
6
6
|
Project-URL: Homepage, https://github.com/NCAR/rda-python-dsquasar
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{rda_python_dsquasar-1.0.9 → rda_python_dsquasar-1.0.10}/src/rda_python_dsquasar/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|