rda-python-miscs 3.0.4__py3-none-any.whl → 3.0.6__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_miscs/decsdata_restore.py +456 -0
- rda_python_miscs/decsdata_restore.usg +101 -0
- rda_python_miscs/decsdata_storage.py +196 -0
- rda_python_miscs/decsdata_storage.usg +66 -0
- rda_python_miscs/gdexcp.py +1 -1
- rda_python_miscs/gdexdrop.py +285 -0
- rda_python_miscs/gdexdrop.usg +77 -0
- rda_python_miscs/gdexdrop_standalone.py +23 -0
- rda_python_miscs/gdexkill.py +1 -1
- rda_python_miscs/gdexmod.py +1 -1
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/METADATA +60 -3
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/RECORD +16 -9
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/WHEEL +1 -1
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/entry_points.txt +3 -0
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/licenses/LICENSE +0 -0
- {rda_python_miscs-3.0.4.dist-info → rda_python_miscs-3.0.6.dist-info}/top_level.txt +0 -0
|
@@ -0,0 +1,456 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
##################################################################################
|
|
3
|
+
# Title: decsdata_restore
|
|
4
|
+
# Author: Zaihua Ji, zji@ucar.edu
|
|
5
|
+
# Date: 2026-08-31
|
|
6
|
+
# Purpose: restore decsdata datasets, or parts of them, out of the GLADE HSM
|
|
7
|
+
# cold storage; the opposite of decsdata_storage
|
|
8
|
+
# Github: https://github.com/NCAR/rda-python-miscs.git
|
|
9
|
+
##################################################################################
|
|
10
|
+
import re
|
|
11
|
+
import os
|
|
12
|
+
import sys
|
|
13
|
+
from os import path as op
|
|
14
|
+
from rda_python_common.pg_file import PgFile
|
|
15
|
+
|
|
16
|
+
class DecsRestore(PgFile):
|
|
17
|
+
"""Restore decsdata datasets out of the GLADE HSM cold storage.
|
|
18
|
+
|
|
19
|
+
'glade_hsm recall' only submits a request that the HSM batch processes
|
|
20
|
+
fulfill later on, so restoring is done in three steps:
|
|
21
|
+
|
|
22
|
+
-x submit the recall requests for the given datasets/paths
|
|
23
|
+
-s check the recall status, repeat until nothing is left on tape
|
|
24
|
+
-r copy the recalled data back under the decsdata directory
|
|
25
|
+
|
|
26
|
+
Recalled files stay readable inside cold storage for 7 days only, after
|
|
27
|
+
which they are migrated onto tape again, so step -r must be done within
|
|
28
|
+
that window.
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self):
|
|
32
|
+
"""Initialize DecsRestore with default option values and runtime state."""
|
|
33
|
+
super().__init__()
|
|
34
|
+
self.HSM = os.environ.get('GLADE_HSM', op.expanduser('~benkirk/glade_hsm'))
|
|
35
|
+
self.VALOPTS = 'Dwl' # single-value options
|
|
36
|
+
self.MULOPTS = 'd' # multi-value options
|
|
37
|
+
self.MODOPTS = 'hf' # mode options
|
|
38
|
+
self.ACTOPTS = 'xsr' # action options, one and only one is required
|
|
39
|
+
self.OPTS = {
|
|
40
|
+
'D': None, # cold storage date, in YYYYMMDD or YYYY-MM-DD
|
|
41
|
+
'w': None, # decsdata directory, defaults to PGLOG['DECSHOME']
|
|
42
|
+
'l': None, # dataset list file
|
|
43
|
+
'd': [], # dataset IDs, a sub-path may be appended to each one
|
|
44
|
+
't': None, # target directory of -r, defaults to the decsdata directory
|
|
45
|
+
'h': 0, # 1 to show help message
|
|
46
|
+
'f': 0, # 1 to copy back while files are still on tape
|
|
47
|
+
}
|
|
48
|
+
self.ACTION = None # one of the ACTOPTS letters
|
|
49
|
+
self.SIZEUNITS = { # units reported by 'gladequota', in bytes
|
|
50
|
+
'B': 1, 'KIB': 1024, 'MIB': 1024**2,
|
|
51
|
+
'GIB': 1024**3, 'TIB': 1024**4, 'PIB': 1024**5,
|
|
52
|
+
}
|
|
53
|
+
self.RINFO = {
|
|
54
|
+
'decsdir': None, # decsdata directory the data is restored into
|
|
55
|
+
'roots': [], # cold storage directories to look the data up in
|
|
56
|
+
'acnt': 0, # number of dataset paths acted on successfully
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
# function to read parameters
|
|
60
|
+
def read_parameters(self):
|
|
61
|
+
"""Parse the command line into the OPTS option values and the action.
|
|
62
|
+
|
|
63
|
+
Single-value options -D, -w and -l take one value each, multi-value
|
|
64
|
+
option -d gathers every following dataset path, mode options -h and -f
|
|
65
|
+
are simple flags, and action options -x, -s and -r are mutually
|
|
66
|
+
exclusive; -r optionally takes a target directory. Exits with usage if
|
|
67
|
+
-h is given or no action is specified.
|
|
68
|
+
"""
|
|
69
|
+
self.set_suid(self.PGLOG['EUID'])
|
|
70
|
+
self.set_help_path(__file__)
|
|
71
|
+
self.PGLOG['LOGFILE'] = "decsdata_restore.log" # set different log file
|
|
72
|
+
argv = sys.argv[1:]
|
|
73
|
+
self.cmdlog("decsdata_restore {}".format(' '.join(argv)))
|
|
74
|
+
option = None
|
|
75
|
+
for arg in argv:
|
|
76
|
+
ms = re.match(r'^-(\w+)$', arg)
|
|
77
|
+
if ms:
|
|
78
|
+
option = ms.group(1)
|
|
79
|
+
if option in self.ACTOPTS:
|
|
80
|
+
self.set_action(option)
|
|
81
|
+
if option != 'r': option = None # -r may be followed by a target directory
|
|
82
|
+
elif option in self.MODOPTS:
|
|
83
|
+
self.OPTS[option] = 1
|
|
84
|
+
option = None
|
|
85
|
+
elif option not in self.VALOPTS and option not in self.MULOPTS:
|
|
86
|
+
self.pglog(arg + ": Unknown Option", self.LGEREX)
|
|
87
|
+
continue
|
|
88
|
+
if not option: self.pglog(arg + ": Value provided without option", self.LGEREX)
|
|
89
|
+
if option in self.MULOPTS:
|
|
90
|
+
self.OPTS[option].append(arg) # gather all values until the next option
|
|
91
|
+
else:
|
|
92
|
+
if option == 'r': option = 't' # the value of -r is the target directory
|
|
93
|
+
self.OPTS[option] = arg
|
|
94
|
+
option = None
|
|
95
|
+
if self.OPTS['h'] or not self.ACTION: self.show_usage("decsdata_restore")
|
|
96
|
+
|
|
97
|
+
# remember the action option and reject a second one
|
|
98
|
+
def set_action(self, option):
|
|
99
|
+
"""Record the single action option to perform.
|
|
100
|
+
|
|
101
|
+
Args:
|
|
102
|
+
option (str): One of the ACTOPTS letters.
|
|
103
|
+
"""
|
|
104
|
+
if self.ACTION and self.ACTION != option:
|
|
105
|
+
self.pglog("-{}: Cannot combine with Action -{}".format(option, self.ACTION), self.LGEREX)
|
|
106
|
+
self.ACTION = option
|
|
107
|
+
|
|
108
|
+
# function to start actions
|
|
109
|
+
def start_actions(self):
|
|
110
|
+
"""Validate the caller, resolve the cold storage paths, and act on each dataset path."""
|
|
111
|
+
self.dssdb_dbname()
|
|
112
|
+
self.validate_decs_group('decsdata_restore', self.PGLOG['CURUID'], 1)
|
|
113
|
+
self.set_restore_paths()
|
|
114
|
+
specs = self.get_dataset_list()
|
|
115
|
+
if not specs: self.pglog("No dataset found to restore", self.LGWNEX)
|
|
116
|
+
if self.ACTION == 'x': self.check_restore_space(specs)
|
|
117
|
+
for spec in specs:
|
|
118
|
+
self.restore_one_path(spec)
|
|
119
|
+
acts = {'x': 'requested Recall', 's': 'checked Status', 'r': 'copied back'}
|
|
120
|
+
s = ('s' if self.RINFO['acnt'] > 1 else '')
|
|
121
|
+
self.pglog("{} of {} Dataset Path{} {}".format(self.RINFO['acnt'], len(specs),
|
|
122
|
+
s, acts[self.ACTION]), self.LOGWRN)
|
|
123
|
+
self.cmdlog()
|
|
124
|
+
|
|
125
|
+
# resolve the decsdata directory and the cold storage directories
|
|
126
|
+
def set_restore_paths(self):
|
|
127
|
+
"""Fill RINFO with the decsdata directory and the cold storage directories to search.
|
|
128
|
+
|
|
129
|
+
For a given -D date only '<decsdata>/cold_storage_<date>/COLD_STORAGE' is
|
|
130
|
+
searched. Without -D both '<decsdata>/COLD_STORAGE' and every
|
|
131
|
+
'<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are searched, the most
|
|
132
|
+
recent dated one first.
|
|
133
|
+
"""
|
|
134
|
+
decsdir = self.OPTS['w'] if self.OPTS['w'] else self.PGLOG['DECSHOME']
|
|
135
|
+
if not self.check_local_file(decsdir, 0, self.LOGWRN):
|
|
136
|
+
self.pglog(decsdir + ": decsdata directory NOT exists", self.LGEREX)
|
|
137
|
+
self.RINFO['decsdir'] = decsdir
|
|
138
|
+
if not self.OPTS['t']: self.OPTS['t'] = decsdir
|
|
139
|
+
roots = []
|
|
140
|
+
if self.OPTS['D']:
|
|
141
|
+
date = re.sub('-', '', self.OPTS['D'])
|
|
142
|
+
if not re.match(r'^\d{8}$', date):
|
|
143
|
+
self.pglog(date + ": Invalid cold storage date, YYYYMMDD expected", self.LGEREX)
|
|
144
|
+
roots.append(self.join_paths(decsdir, "cold_storage_{}/COLD_STORAGE".format(date)))
|
|
145
|
+
else:
|
|
146
|
+
roots.append(self.join_paths(decsdir, "COLD_STORAGE"))
|
|
147
|
+
files = self.local_glob(self.join_paths(decsdir, "cold_storage_[0-9]*/COLD_STORAGE"), 0, self.LOGWRN)
|
|
148
|
+
for file in sorted(files, reverse = True):
|
|
149
|
+
if not files[file]['isfile']: roots.append(file)
|
|
150
|
+
for root in roots:
|
|
151
|
+
if self.check_local_file(root, 0, 0): self.RINFO['roots'].append(root)
|
|
152
|
+
if not self.RINFO['roots']:
|
|
153
|
+
self.pglog("{}: No cold storage directory found in {}".format(', '.join(roots), decsdir), self.LGEREX)
|
|
154
|
+
self.pglog("Cold storage searched: {}".format(', '.join(self.RINFO['roots'])), self.LOGWRN)
|
|
155
|
+
|
|
156
|
+
# gather the dataset paths to restore
|
|
157
|
+
def get_dataset_list(self):
|
|
158
|
+
"""Return the list of dataset paths to restore.
|
|
159
|
+
|
|
160
|
+
Uses the -d values if given. Otherwise reads the -l list file, creating
|
|
161
|
+
it first from every dNNNNNN directory in the cold storage directories if
|
|
162
|
+
it does not exist yet.
|
|
163
|
+
|
|
164
|
+
Returns:
|
|
165
|
+
list[str]: Dataset IDs, each optionally followed by a sub-path.
|
|
166
|
+
"""
|
|
167
|
+
if self.OPTS['d']:
|
|
168
|
+
self.pglog("Restore {} given Dataset Path(s)".format(len(self.OPTS['d'])), self.LOGWRN)
|
|
169
|
+
return self.OPTS['d']
|
|
170
|
+
lstfile = self.OPTS['l'] if self.OPTS['l'] else "dsids_{}.lst".format(re.sub('-', '', self.curdate()))
|
|
171
|
+
if not op.isfile(lstfile):
|
|
172
|
+
dsids = self.get_coldstorage_datasets()
|
|
173
|
+
with open(lstfile, 'w') as OUT:
|
|
174
|
+
for dsid in dsids: OUT.write(dsid + "\n")
|
|
175
|
+
self.pglog("{}: Generated with {} Dataset(s)".format(lstfile, len(dsids)), self.LOGWRN)
|
|
176
|
+
specs = []
|
|
177
|
+
with open(lstfile, 'r') as IN:
|
|
178
|
+
for line in IN:
|
|
179
|
+
line = line.strip()
|
|
180
|
+
if line: specs.append(line)
|
|
181
|
+
self.pglog("{}: Read {} Dataset Path(s) to restore".format(lstfile, len(specs)), self.LOGWRN)
|
|
182
|
+
return specs
|
|
183
|
+
|
|
184
|
+
# find all dNNNNNN dataset directories in the cold storage directories
|
|
185
|
+
def get_coldstorage_datasets(self):
|
|
186
|
+
"""Return the sorted dataset IDs of every dNNNNNN directory in the cold storage directories.
|
|
187
|
+
|
|
188
|
+
Returns:
|
|
189
|
+
list[str]: Unique dataset IDs; plain files matching the pattern are skipped.
|
|
190
|
+
"""
|
|
191
|
+
dsids = []
|
|
192
|
+
for root in self.RINFO['roots']:
|
|
193
|
+
files = self.local_glob(self.join_paths(root, "d" + "[0-9]"*6), 0, self.LOGWRN)
|
|
194
|
+
for file in files:
|
|
195
|
+
if files[file]['isfile']: continue
|
|
196
|
+
dsid = op.basename(file)
|
|
197
|
+
if dsid not in dsids: dsids.append(dsid)
|
|
198
|
+
return sorted(dsids)
|
|
199
|
+
|
|
200
|
+
# locate a dataset path in the cold storage directories, the first match wins
|
|
201
|
+
def find_cold_path(self, spec):
|
|
202
|
+
"""Look a dataset path up in each cold storage directory.
|
|
203
|
+
|
|
204
|
+
Warns and names the ignored ones if the path is found in more than one
|
|
205
|
+
cold storage directory.
|
|
206
|
+
|
|
207
|
+
Args:
|
|
208
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
209
|
+
|
|
210
|
+
Returns:
|
|
211
|
+
tuple: (path, info dict) of the first match, or (None, None).
|
|
212
|
+
"""
|
|
213
|
+
hits = {}
|
|
214
|
+
for root in self.RINFO['roots']:
|
|
215
|
+
path = self.join_paths(root, spec)
|
|
216
|
+
info = self.check_local_file(path, 0, 0)
|
|
217
|
+
if info: hits[path] = info
|
|
218
|
+
if not hits: return (None, None)
|
|
219
|
+
paths = list(hits)
|
|
220
|
+
if len(paths) > 1:
|
|
221
|
+
self.pglog("{}: Found in {} cold storage directories, use {}".format(spec, len(paths), paths[0]), self.LOGWRN)
|
|
222
|
+
self.pglog("{}: Ignored {}".format(spec, ', '.join(paths[1:])), self.LOGWRN)
|
|
223
|
+
return (paths[0], hits[paths[0]])
|
|
224
|
+
|
|
225
|
+
# count the files of a cold storage path still on tape
|
|
226
|
+
def hsm_offline_count(self, path, isfile):
|
|
227
|
+
"""Return the number of files under a cold storage path that are still on tape.
|
|
228
|
+
|
|
229
|
+
Args:
|
|
230
|
+
path (str): Cold storage path of a file or directory.
|
|
231
|
+
isfile (int): 1 if path is a regular file, 0 for a directory.
|
|
232
|
+
|
|
233
|
+
Returns:
|
|
234
|
+
int | None: Count of offline files, or None if it cannot be determined.
|
|
235
|
+
"""
|
|
236
|
+
out = self.pgsystem("{} status {}".format(self.HSM, path), self.LOGWRN, 51)
|
|
237
|
+
if not out: return None
|
|
238
|
+
if isfile: return (1 if re.search(r'migrated', out) else 0)
|
|
239
|
+
cnts = re.findall(r'Offline:\s*([\d,]+)', out)
|
|
240
|
+
if not cnts: return None
|
|
241
|
+
return int(re.sub(',', '', cnts[-1]))
|
|
242
|
+
|
|
243
|
+
# build the sfile condition of one dataset path
|
|
244
|
+
def sfile_condition(self, spec):
|
|
245
|
+
"""Turn a dataset path into a condition on table dssdb.sfile.
|
|
246
|
+
|
|
247
|
+
A saved file lives in '<decsdata>/<dsid>/<type>/<sfile>', so the first
|
|
248
|
+
component of the path is the dataset ID, the second one the saved file
|
|
249
|
+
type, and the rest the leading part of the sfile field.
|
|
250
|
+
|
|
251
|
+
Args:
|
|
252
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
253
|
+
|
|
254
|
+
Returns:
|
|
255
|
+
str: The WHERE condition of the saved files under the path.
|
|
256
|
+
"""
|
|
257
|
+
paths = spec.strip('/').split('/')
|
|
258
|
+
if not re.match(r'^[a-z]\d{6}$', paths[0]):
|
|
259
|
+
self.pglog(spec + ": Invalid dataset path, dNNNNNN expected", self.LGEREX)
|
|
260
|
+
cnd = "dsid = '{}'".format(paths[0])
|
|
261
|
+
if len(paths) > 1:
|
|
262
|
+
if not re.match(r'^\w$', paths[1]):
|
|
263
|
+
self.pglog(spec + ": Invalid saved file type, one word character expected", self.LGEREX)
|
|
264
|
+
cnd += " AND type = '{}'".format(paths[1])
|
|
265
|
+
if len(paths) > 2:
|
|
266
|
+
sfile = '/'.join(paths[2:])
|
|
267
|
+
if re.search(r"['\\]", sfile):
|
|
268
|
+
self.pglog(spec + ": Invalid saved file path", self.LGEREX)
|
|
269
|
+
# the path is either a saved file itself or the directory holding them
|
|
270
|
+
cnd += " AND (sfile = '{}' OR sfile LIKE '{}/%')".format(sfile, re.sub(r'([%_])', r'\\\1', sfile))
|
|
271
|
+
return cnd
|
|
272
|
+
|
|
273
|
+
# get the archived size of one dataset path
|
|
274
|
+
def restore_data_size(self, spec):
|
|
275
|
+
"""Return the total size of the saved files under one dataset path.
|
|
276
|
+
|
|
277
|
+
Args:
|
|
278
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
279
|
+
|
|
280
|
+
Returns:
|
|
281
|
+
int: Number of bytes recorded in dssdb.sfile; 0 if nothing is found.
|
|
282
|
+
"""
|
|
283
|
+
pgrec = self.pgget('sfile', "sum(data_size) tsize, count(sid) fcnt",
|
|
284
|
+
self.sfile_condition(spec), self.LOGWRN)
|
|
285
|
+
if not pgrec or not pgrec['tsize']:
|
|
286
|
+
self.pglog(spec + ": No saved file found in RDADB", self.LOGWRN)
|
|
287
|
+
return 0
|
|
288
|
+
self.pglog("{}: {} in {} saved file(s)".format(spec,
|
|
289
|
+
self.format_float_value(pgrec['tsize']), pgrec['fcnt']), self.LOGWRN)
|
|
290
|
+
return int(pgrec['tsize'])
|
|
291
|
+
|
|
292
|
+
# get the GLADE space left for the decsdata directory
|
|
293
|
+
def decsdata_free_size(self):
|
|
294
|
+
"""Return the GLADE space left on the quota holding the decsdata directory.
|
|
295
|
+
|
|
296
|
+
Parses the 'Used' and 'Quota' columns of 'gladequota' and picks the
|
|
297
|
+
longest reported path the decsdata directory falls under.
|
|
298
|
+
|
|
299
|
+
Returns:
|
|
300
|
+
int | None: Number of free bytes, or None if it cannot be determined.
|
|
301
|
+
"""
|
|
302
|
+
cmd = self.get_local_command("gladequota", self.PGLOG['COMMONUSER'])
|
|
303
|
+
out = self.pgsystem(cmd, self.LOGWRN, 21) # 1+4+16, log the command and return stdout
|
|
304
|
+
if not out: return None
|
|
305
|
+
target = op.realpath(self.RINFO['decsdir'])
|
|
306
|
+
(fsize, fpath) = (None, None)
|
|
307
|
+
for line in out.split('\n'):
|
|
308
|
+
ms = re.match(r'^(/\S+)\s+([\d.]+)\s*(\w+)\s+([\d.]+)\s*(\w+)', line)
|
|
309
|
+
if not ms: continue # skips the header and the 'n/a' quota lines
|
|
310
|
+
path = ms.group(1)
|
|
311
|
+
if not (target == path or target.startswith(path + '/')): continue
|
|
312
|
+
if fpath and len(fpath) >= len(path): continue # keeps the closest path only
|
|
313
|
+
used = self.quota_size(ms.group(2), ms.group(3))
|
|
314
|
+
quota = self.quota_size(ms.group(4), ms.group(5))
|
|
315
|
+
if used is None or quota is None: continue
|
|
316
|
+
(fsize, fpath) = (max(quota - used, 0), path)
|
|
317
|
+
return fsize
|
|
318
|
+
|
|
319
|
+
# convert one 'gladequota' size into bytes
|
|
320
|
+
def quota_size(self, value, unit):
|
|
321
|
+
"""Convert one size reported by 'gladequota' into bytes.
|
|
322
|
+
|
|
323
|
+
Args:
|
|
324
|
+
value (str): The numeric part of the size.
|
|
325
|
+
unit (str): The unit of the size, such as 'TiB'.
|
|
326
|
+
|
|
327
|
+
Returns:
|
|
328
|
+
int | None: Number of bytes, or None for an unknown unit.
|
|
329
|
+
"""
|
|
330
|
+
unit = unit.upper()
|
|
331
|
+
if unit not in self.SIZEUNITS: return None
|
|
332
|
+
return int(float(value)*self.SIZEUNITS[unit])
|
|
333
|
+
|
|
334
|
+
# make sure the decsdata directory has room for the whole restore
|
|
335
|
+
def check_restore_space(self, specs):
|
|
336
|
+
"""Stop the recall if the decsdata directory cannot hold the whole restore.
|
|
337
|
+
|
|
338
|
+
The size to restore is added up from table dssdb.sfile and compared to
|
|
339
|
+
the GLADE space left for the decsdata directory. Twice the size is
|
|
340
|
+
required, since the recall brings the data back on disk inside the cold
|
|
341
|
+
storage first and Action -r copies it back afterwards, so both copies
|
|
342
|
+
live under the decsdata quota at the same time. The check is skipped,
|
|
343
|
+
with a warning, if either size cannot be determined.
|
|
344
|
+
|
|
345
|
+
Args:
|
|
346
|
+
specs (list[str]): Dataset paths to recall.
|
|
347
|
+
"""
|
|
348
|
+
tsize = 0
|
|
349
|
+
for spec in specs:
|
|
350
|
+
tsize += self.restore_data_size(spec)
|
|
351
|
+
if not tsize:
|
|
352
|
+
self.pglog("Unknown size to restore, Skip checking the decsdata space", self.LOGWRN)
|
|
353
|
+
return
|
|
354
|
+
fsize = self.decsdata_free_size()
|
|
355
|
+
if fsize is None:
|
|
356
|
+
self.pglog("{}: Cannot get the space left, Skip checking the decsdata space".format(self.RINFO['decsdir']), self.LOGWRN)
|
|
357
|
+
return
|
|
358
|
+
nsize = 2*tsize # room for the recalled copy and for the copy of Action -r
|
|
359
|
+
msg = "{}: Restore {}, needs {} of the {} left".format(self.RINFO['decsdir'],
|
|
360
|
+
self.format_float_value(tsize), self.format_float_value(nsize),
|
|
361
|
+
self.format_float_value(fsize))
|
|
362
|
+
if nsize > fsize:
|
|
363
|
+
self.pglog(msg + ", NOT enough space", self.LGEREX)
|
|
364
|
+
self.pglog(msg, self.LOGWRN)
|
|
365
|
+
|
|
366
|
+
# act on one dataset path in cold storage
|
|
367
|
+
def restore_one_path(self, spec):
|
|
368
|
+
"""Perform the requested action on one dataset path in cold storage.
|
|
369
|
+
|
|
370
|
+
Args:
|
|
371
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
372
|
+
"""
|
|
373
|
+
(path, info) = self.find_cold_path(spec)
|
|
374
|
+
if not path:
|
|
375
|
+
self.pglog(spec + ": NOT found in cold storage", self.LOGERR)
|
|
376
|
+
return
|
|
377
|
+
if self.ACTION == 'x':
|
|
378
|
+
self.recall_cold_path(spec, path)
|
|
379
|
+
elif self.ACTION == 's':
|
|
380
|
+
self.status_cold_path(spec, path, info['isfile'])
|
|
381
|
+
else:
|
|
382
|
+
self.copy_cold_path(spec, path, info['isfile'])
|
|
383
|
+
|
|
384
|
+
# submit the recall request of one cold storage path
|
|
385
|
+
def recall_cold_path(self, spec, path):
|
|
386
|
+
"""Submit the HSM recall request for one cold storage path.
|
|
387
|
+
|
|
388
|
+
Args:
|
|
389
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
390
|
+
path (str): Cold storage path of the data.
|
|
391
|
+
"""
|
|
392
|
+
if self.pgsystem("{} recall -f {}".format(self.HSM, path), self.LOGWRN, 7):
|
|
393
|
+
self.RINFO['acnt'] += 1
|
|
394
|
+
self.pglog("{}: Recall requested, check the progress via -s".format(spec), self.LOGWRN)
|
|
395
|
+
else:
|
|
396
|
+
self.pglog("{}: Error request Recall of {}".format(spec, path), self.LOGERR)
|
|
397
|
+
|
|
398
|
+
# report the recall status of one cold storage path
|
|
399
|
+
def status_cold_path(self, spec, path, isfile):
|
|
400
|
+
"""Report the HSM and recall status of one cold storage path.
|
|
401
|
+
|
|
402
|
+
Args:
|
|
403
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
404
|
+
path (str): Cold storage path of the data.
|
|
405
|
+
isfile (int): 1 if path is a regular file, 0 for a directory.
|
|
406
|
+
"""
|
|
407
|
+
offline = self.hsm_offline_count(path, isfile)
|
|
408
|
+
if offline is None:
|
|
409
|
+
self.pglog("{}: Cannot get the offline file count of {}".format(spec, path), self.LOGERR)
|
|
410
|
+
return
|
|
411
|
+
self.RINFO['acnt'] += 1
|
|
412
|
+
if offline > 0:
|
|
413
|
+
s = ('s' if offline > 1 else '')
|
|
414
|
+
self.pglog("{}: Recall PENDING, {} File{} still on tape".format(spec, offline, s), self.LOGWRN)
|
|
415
|
+
else:
|
|
416
|
+
self.pglog("{}: Recall COMPLETE, ready to copy back via -r".format(spec), self.LOGWRN)
|
|
417
|
+
|
|
418
|
+
# copy one recalled cold storage path back into the decsdata directory
|
|
419
|
+
def copy_cold_path(self, spec, path, isfile):
|
|
420
|
+
"""Copy one recalled cold storage path back to its target directory.
|
|
421
|
+
|
|
422
|
+
Nothing is copied while files are still on tape unless -f is given.
|
|
423
|
+
|
|
424
|
+
Args:
|
|
425
|
+
spec (str): Dataset ID, optionally followed by a sub-path.
|
|
426
|
+
path (str): Cold storage path of the data.
|
|
427
|
+
isfile (int): 1 if path is a regular file, 0 for a directory.
|
|
428
|
+
"""
|
|
429
|
+
offline = self.hsm_offline_count(path, isfile)
|
|
430
|
+
if offline:
|
|
431
|
+
s = ('s' if offline > 1 else '')
|
|
432
|
+
if not self.OPTS['f']:
|
|
433
|
+
self.pglog("{}: {} File{} still on tape, add Mode -f to copy anyway".format(spec, offline, s), self.LOGERR)
|
|
434
|
+
return
|
|
435
|
+
self.pglog("{}: {} File{} still on tape, copy it anyway".format(spec, offline, s), self.LOGWRN)
|
|
436
|
+
tofile = self.join_paths(self.OPTS['t'], spec)
|
|
437
|
+
# a directory is copied as '<path>/.' to merge into an existing target
|
|
438
|
+
fromfile = path if isfile else path + "/."
|
|
439
|
+
if self.local_copy_local(tofile, fromfile, self.LOGWRN):
|
|
440
|
+
self.RINFO['acnt'] += 1
|
|
441
|
+
self.pglog("{}: Copied back to {}".format(spec, tofile), self.LOGWRN)
|
|
442
|
+
else:
|
|
443
|
+
self.pglog("{}: Error copy {} back to {}".format(spec, path, tofile), self.LOGERR)
|
|
444
|
+
|
|
445
|
+
# main function to execute this script
|
|
446
|
+
def main():
|
|
447
|
+
"""Entry point: instantiate DecsRestore, parse arguments, run, and exit."""
|
|
448
|
+
from rda_python_setuid.setup_guide import show_setup_guide
|
|
449
|
+
object = DecsRestore()
|
|
450
|
+
show_setup_guide(object, 'rda_python_miscs', ['decsdata_storage', 'decsdata_restore'])
|
|
451
|
+
object.read_parameters()
|
|
452
|
+
object.start_actions()
|
|
453
|
+
object.pgexit(0)
|
|
454
|
+
|
|
455
|
+
# call main() to start program
|
|
456
|
+
if __name__ == "__main__": main()
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
|
|
2
|
+
Restore decsdata datasets, or parts of them, out of the GLADE HSM cold storage.
|
|
3
|
+
This is the opposite of 'decsdata_storage'. A recall request is only submitted
|
|
4
|
+
by 'glade_hsm recall' and is fulfilled by the HSM batch processes later on, so
|
|
5
|
+
restoring is done in three steps:
|
|
6
|
+
|
|
7
|
+
1. Option -x submits the recall requests for the given dataset paths;
|
|
8
|
+
2. Option -s checks the recall status, repeat it until nothing is on tape;
|
|
9
|
+
3. Option -r copies the recalled data back into the decsdata directory.
|
|
10
|
+
|
|
11
|
+
Recalled files stay readable inside the 'COLD_STORAGE/' path for 7 days only,
|
|
12
|
+
after which they are migrated onto tape again, so step 3 must be done within
|
|
13
|
+
that window.
|
|
14
|
+
|
|
15
|
+
Usage: decsdata_restore [-D Date] [-w DecsdataDirectory] [-l ListFile] \
|
|
16
|
+
[-d DatasetPathList] [-f] -x|-s|-r [TargetDirectory]
|
|
17
|
+
|
|
18
|
+
- Option -D Date, the cold storage date, in YYYYMMDD or YYYY-MM-DD. Only
|
|
19
|
+
'<decsdata>/cold_storage_<date>/COLD_STORAGE' is searched for the
|
|
20
|
+
data. Without it, '<decsdata>/COLD_STORAGE' and every
|
|
21
|
+
'<decsdata>/cold_storage_<YYYYMMDD>/COLD_STORAGE' are all searched,
|
|
22
|
+
the most recent dated one first. A dataset path found in more than
|
|
23
|
+
one of them is acted on in the first one only, and the ignored ones
|
|
24
|
+
are logged;
|
|
25
|
+
|
|
26
|
+
- Option -w DecsdataDirectory, the decsdata directory holding the cold
|
|
27
|
+
storage directories, and the directory the data is copied back into.
|
|
28
|
+
Defaults to the configured decsdata root path, /gdex/decsdata;
|
|
29
|
+
|
|
30
|
+
- Option -l ListFile, a file holding one dataset path per line. Only
|
|
31
|
+
used if -d is not given. Defaults to 'dsids_<today>.lst' in the
|
|
32
|
+
current directory, which is generated from every dNNNNNN directory
|
|
33
|
+
found in the cold storage directories if it does not exist yet;
|
|
34
|
+
|
|
35
|
+
- Option -d DatasetPathList, one or more dataset IDs to restore, such as
|
|
36
|
+
'-d d612000 d627000'. Append a sub-path to a dataset ID to restore
|
|
37
|
+
part of it only, such as '-d d612000/2020/01';
|
|
38
|
+
|
|
39
|
+
- Option -f, copy the data back even while some of its files are still on
|
|
40
|
+
tape. Without it, Action -r refuses to copy such data;
|
|
41
|
+
|
|
42
|
+
- Option -h, display this help document;
|
|
43
|
+
|
|
44
|
+
- Action -x, submit the recall requests to bring the data back on disk.
|
|
45
|
+
The decsdata area is limited, so the size to restore is added up
|
|
46
|
+
from Table 'sfile' in RDADB and compared to the space left for the
|
|
47
|
+
decsdata directory, as reported by 'gladequota', before any recall
|
|
48
|
+
is requested. TWICE the size is required, since the recalled copy
|
|
49
|
+
in the cold storage and the copy made by Action -r later on both
|
|
50
|
+
live under the decsdata quota; the recall is stopped if the space
|
|
51
|
+
left is not enough;
|
|
52
|
+
|
|
53
|
+
- Action -s, report the HSM status of the data, including the number of
|
|
54
|
+
files still on tape and the log of any outstanding recall request;
|
|
55
|
+
|
|
56
|
+
- Action -r [TargetDirectory], copy the recalled data back under
|
|
57
|
+
'<decsdata>/<dsid>', or under the given TargetDirectory. A restored
|
|
58
|
+
sub-path keeps its relative position, so 'd612000/2020/01' is copied
|
|
59
|
+
back to '<TargetDirectory>/d612000/2020/01'. The dataset directory,
|
|
60
|
+
and any sub-directory of it, is created if it does not exist yet.
|
|
61
|
+
|
|
62
|
+
One and only one of the Actions -x, -s and -r is required; this help document
|
|
63
|
+
is displayed without any of them. This utility can be run from any directory.
|
|
64
|
+
It is executed under the effective user 'gdexdata' via setuid, so the restored
|
|
65
|
+
data is owned by 'gdexdata'. The cold storage copy of the data is left in
|
|
66
|
+
place by Action -r; move it out of the 'COLD_STORAGE/' path manually to take
|
|
67
|
+
a dataset out of the HSM permanently.
|
|
68
|
+
|
|
69
|
+
Examples:
|
|
70
|
+
|
|
71
|
+
1. Restore a whole dataset from the cold storage of the root decsdata
|
|
72
|
+
directory, /gdex/decsdata/COLD_STORAGE/d612000:
|
|
73
|
+
|
|
74
|
+
decsdata_restore -d d612000 -x
|
|
75
|
+
decsdata_restore -d d612000 -s
|
|
76
|
+
decsdata_restore -d d612000 -r
|
|
77
|
+
|
|
78
|
+
2. Restore it from the cold storage directory of a specific date,
|
|
79
|
+
/gdex/decsdata/cold_storage_20250529/COLD_STORAGE/d612000:
|
|
80
|
+
|
|
81
|
+
decsdata_restore -D 20250529 -d d612000 -x
|
|
82
|
+
|
|
83
|
+
3. Restore one year of a dataset only:
|
|
84
|
+
|
|
85
|
+
decsdata_restore -d d612000/2020 -x
|
|
86
|
+
decsdata_restore -d d612000/2020 -s
|
|
87
|
+
decsdata_restore -d d612000/2020 -r
|
|
88
|
+
|
|
89
|
+
4. Copy the recalled data back to a directory other than the decsdata one:
|
|
90
|
+
|
|
91
|
+
decsdata_restore -d d612000 -r /PathTo/OtherDirectory
|
|
92
|
+
|
|
93
|
+
5. Check the status of every dataset in every cold storage directory,
|
|
94
|
+
generating the dataset list file automatically:
|
|
95
|
+
|
|
96
|
+
decsdata_restore -s
|
|
97
|
+
|
|
98
|
+
6. Copy a dataset back even though some of its files are still on tape,
|
|
99
|
+
which makes the HSM read them off tape while they are being copied:
|
|
100
|
+
|
|
101
|
+
decsdata_restore -d d612000 -f -r
|