rda-python-dsarch 2.0.5__py3-none-any.whl → 2.0.9__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- rda_python_dsarch/dsarch.py +666 -83
- rda_python_dsarch/dsarch.usg +1382 -1717
- rda_python_dsarch/pg_arch.py +500 -71
- rda_python_dsarch/pg_meta.py +471 -53
- {rda_python_dsarch-2.0.5.dist-info → rda_python_dsarch-2.0.9.dist-info}/METADATA +1 -1
- rda_python_dsarch-2.0.9.dist-info/RECORD +14 -0
- {rda_python_dsarch-2.0.5.dist-info → rda_python_dsarch-2.0.9.dist-info}/WHEEL +1 -1
- rda_python_dsarch-2.0.5.dist-info/RECORD +0 -14
- {rda_python_dsarch-2.0.5.dist-info → rda_python_dsarch-2.0.9.dist-info}/entry_points.txt +0 -0
- {rda_python_dsarch-2.0.5.dist-info → rda_python_dsarch-2.0.9.dist-info}/licenses/LICENSE +0 -0
- {rda_python_dsarch-2.0.5.dist-info → rda_python_dsarch-2.0.9.dist-info}/top_level.txt +0 -0
rda_python_dsarch/dsarch.py
CHANGED
|
@@ -10,6 +10,16 @@
|
|
|
10
10
|
# server; and save information of data files into RDADB
|
|
11
11
|
# Github: https://github.com/NCAR/rda-python-dsarch.git
|
|
12
12
|
##################################################################################
|
|
13
|
+
"""
|
|
14
|
+
dsarch.py - Archive utility for GDEX/RDADB dataset file management.
|
|
15
|
+
|
|
16
|
+
Implements the DsArch class which orchestrates all dsarch actions: archiving
|
|
17
|
+
web, help, saved, and Quasar backup files; retrieving, moving, deleting, and
|
|
18
|
+
reordering files; managing dataset, group, and version metadata in RDADB.
|
|
19
|
+
|
|
20
|
+
Entry point:
|
|
21
|
+
dsarch.main()
|
|
22
|
+
"""
|
|
13
23
|
import sys
|
|
14
24
|
import os
|
|
15
25
|
import re
|
|
@@ -18,7 +28,29 @@ from .pg_meta import PgMeta
|
|
|
18
28
|
from .pg_arch import PgArch
|
|
19
29
|
|
|
20
30
|
class DsArch(PgArch, PgMeta):
|
|
31
|
+
"""
|
|
32
|
+
Main action class for the dsarch utility.
|
|
33
|
+
|
|
34
|
+
Inherits shared state and helpers from PgArch and database interaction
|
|
35
|
+
methods from PgMeta. The two primary entry points after construction are:
|
|
36
|
+
|
|
37
|
+
read_parameters() -- parse and validate command-line arguments
|
|
38
|
+
start_actions() -- dispatch to the appropriate action method
|
|
39
|
+
|
|
40
|
+
Instance attributes set in __init__:
|
|
41
|
+
ERRCNT -- cumulative count of recoverable errors in the current action
|
|
42
|
+
RETSTAT -- non-zero when an action finishes with partial failure
|
|
43
|
+
ALLCNT -- total number of files to process in the current action
|
|
44
|
+
ADDCNT -- number of new records added during the current action
|
|
45
|
+
MODCNT -- number of existing records modified during the current action
|
|
46
|
+
OVERRIDE -- non-zero when the -OE (OverrideExist) flag is active
|
|
47
|
+
TARFILES -- dict mapping tar archive paths to their member-file metadata
|
|
48
|
+
VINDEX -- dict mapping version indices to version records
|
|
49
|
+
CHGGRPS -- dict of group indices whose file-number counts need updating
|
|
50
|
+
ERRMSG -- accumulated error message string for the current action
|
|
51
|
+
"""
|
|
21
52
|
def __init__(self):
|
|
53
|
+
"""Initialize DsArch, resetting action counters and working collections."""
|
|
22
54
|
super().__init__() # initialize parent class
|
|
23
55
|
self.ERRCNT = self.RETSTAT = self.ALLCNT = self.ADDCNT = self.MODCNT = self.OVERRIDE = 0
|
|
24
56
|
self.TARFILES = {}
|
|
@@ -26,8 +58,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
26
58
|
self.CHGGRPS = {}
|
|
27
59
|
self.ERRMSG = ''
|
|
28
60
|
|
|
29
|
-
# function to read paramters
|
|
30
61
|
def read_parameters(self):
|
|
62
|
+
"""
|
|
63
|
+
Parse and validate command-line arguments for dsarch.
|
|
64
|
+
|
|
65
|
+
Calls parsing_input() to populate self.params, then attempts to infer
|
|
66
|
+
the dataset ID from file arguments when -DS is not explicitly given
|
|
67
|
+
(supported for -GH, -GW, -GS, -GQ actions). Auto-collects file names
|
|
68
|
+
from the origin group when -GI and -OG are both present for -SS/-SW
|
|
69
|
+
actions. Finishes with check_enough_options() to abort on missing
|
|
70
|
+
required options.
|
|
71
|
+
"""
|
|
31
72
|
self.set_help_path(__file__)
|
|
32
73
|
pgname = "dsarch"
|
|
33
74
|
self.parsing_input(pgname)
|
|
@@ -53,8 +94,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
53
94
|
self.params['WF'] = self.get_filenames("wfile")
|
|
54
95
|
self.check_enough_options(self.PGOPT['CACT'], self.PGOPT['ACTS'])
|
|
55
96
|
|
|
56
|
-
# start actions of dsarch
|
|
57
97
|
def start_actions(self):
|
|
98
|
+
"""
|
|
99
|
+
Dispatch to the appropriate action handler based on the parsed action flag.
|
|
100
|
+
|
|
101
|
+
Routes the current action (stored in self.PGOPT['ACTS']) to the matching
|
|
102
|
+
archive, get, set, move, delete, or metadata method. After the primary
|
|
103
|
+
action, handles cross-cutting concerns:
|
|
104
|
+
- Dataset period updates when date/time params are present
|
|
105
|
+
- File-count resets (-WN), metadata resets (-WM), top-group index
|
|
106
|
+
resets (-RT) for write-mode actions
|
|
107
|
+
- Final success/failure logging and optional email notification
|
|
108
|
+
"""
|
|
58
109
|
setds = 0
|
|
59
110
|
dsid = self.params['DS']
|
|
60
111
|
self.OVERRIDE = self.OVRIDE if 'OE' in self.params else 0
|
|
@@ -302,8 +353,21 @@ class DsArch(PgArch, PgMeta):
|
|
|
302
353
|
else:
|
|
303
354
|
self.cmdlog()
|
|
304
355
|
|
|
305
|
-
# archive web/object files
|
|
306
356
|
def archive_web_files(self):
|
|
357
|
+
"""
|
|
358
|
+
Archive local files as web files on GLADE and/or the object store.
|
|
359
|
+
|
|
360
|
+
For each local file in params['LF']:
|
|
361
|
+
- Determines archive format, location flag, and web path
|
|
362
|
+
- Skips files that are already archived with the same checksum/size
|
|
363
|
+
(unless -OE override is set)
|
|
364
|
+
- Copies to GLADE (warch) and/or object store (oarch) as needed
|
|
365
|
+
- Registers or updates the web file record in RDADB via set_one_webfile()
|
|
366
|
+
- Optionally triggers metadata XML gathering (-GX) or deletion (-DX)
|
|
367
|
+
|
|
368
|
+
Retries the whole batch if transient errors occur, stopping when the
|
|
369
|
+
error count stabilises. Sets self.RETSTAT on unresolved failures.
|
|
370
|
+
"""
|
|
307
371
|
tname = 'wfile'
|
|
308
372
|
dftloc = None
|
|
309
373
|
dsid = self.params['DS']
|
|
@@ -524,8 +588,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
524
588
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
525
589
|
if dslocflags: self.set_dataset_locflag(dsid, dslocflags.pop())
|
|
526
590
|
|
|
527
|
-
# archive help files
|
|
528
591
|
def archive_help_files(self):
|
|
592
|
+
"""
|
|
593
|
+
Archive local files as dataset help/documentation files on GLADE/object store.
|
|
594
|
+
|
|
595
|
+
Mirrors the logic of archive_web_files() for help files, using the help
|
|
596
|
+
file path (RTPATH['H<type>']) and registering records in the hfile table.
|
|
597
|
+
Handles both GLADE (warch) and object-store (oarch) destinations based on
|
|
598
|
+
the file's location flag. Sets self.RETSTAT on unresolved failures.
|
|
599
|
+
"""
|
|
529
600
|
tname = 'hfile'
|
|
530
601
|
dsid = self.params['DS']
|
|
531
602
|
bucket = self.PGLOG['OBJCTBKT']
|
|
@@ -718,8 +789,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
718
789
|
self.reset_rdadb_version(dsid)
|
|
719
790
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
720
791
|
|
|
721
|
-
# archive save files
|
|
722
792
|
def archive_saved_files(self):
|
|
793
|
+
"""
|
|
794
|
+
Archive local files as dataset saved files on GLADE.
|
|
795
|
+
|
|
796
|
+
Copies each local file in params['LF'] to its saved-file path, validates
|
|
797
|
+
type constraints, handles version control checks, and registers or updates
|
|
798
|
+
the saved file record in RDADB via set_one_savedfile(). Retries on
|
|
799
|
+
transient copy failures. Sets self.RETSTAT on unresolved failures.
|
|
800
|
+
"""
|
|
723
801
|
tname = 'sfile'
|
|
724
802
|
dftloc = 'G'
|
|
725
803
|
dsid = self.params['DS']
|
|
@@ -881,8 +959,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
881
959
|
self.reset_rdadb_version(dsid)
|
|
882
960
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
883
961
|
|
|
884
|
-
# cross copy web files between glade and object store
|
|
885
962
|
def crosscopy_web_files(self, aname):
|
|
963
|
+
"""
|
|
964
|
+
Cross-copy (or cross-move) web files between GLADE and the object store.
|
|
965
|
+
|
|
966
|
+
Args:
|
|
967
|
+
aname -- 'Copy' to copy files, 'Move' to move (deletes the source)
|
|
968
|
+
"""
|
|
886
969
|
tname = 'wfile'
|
|
887
970
|
dsid = self.params['DS']
|
|
888
971
|
s = 's' if self.ALLCNT > 1 else ''
|
|
@@ -1029,8 +1112,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
1029
1112
|
self.reset_rdadb_version(dsid)
|
|
1030
1113
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1031
1114
|
|
|
1032
|
-
# cross copy help files between glade and object store
|
|
1033
1115
|
def crosscopy_help_files(self, aname):
|
|
1116
|
+
"""
|
|
1117
|
+
Cross-copy (or cross-move) help files between GLADE and the object store.
|
|
1118
|
+
|
|
1119
|
+
Args:
|
|
1120
|
+
aname -- 'Copy' to copy files, 'Move' to move (deletes the source)
|
|
1121
|
+
"""
|
|
1034
1122
|
tname = 'hfile'
|
|
1035
1123
|
dsid = self.params['DS']
|
|
1036
1124
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1175,8 +1263,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
1175
1263
|
self.reset_rdadb_version(dsid)
|
|
1176
1264
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1177
1265
|
|
|
1178
|
-
# get backup file names in RDADB and on Quaser server for given dataset id
|
|
1179
1266
|
def get_backup_filenames(self, bfile, dsid):
|
|
1267
|
+
"""
|
|
1268
|
+
Retrieve backup file names from RDADB and verify their existence on Quasar.
|
|
1269
|
+
|
|
1270
|
+
Args:
|
|
1271
|
+
bfile -- backup file name or pattern to search for
|
|
1272
|
+
dsid -- dataset ID string
|
|
1273
|
+
|
|
1274
|
+
Returns:
|
|
1275
|
+
List of matching backup file name strings.
|
|
1276
|
+
"""
|
|
1180
1277
|
ms = re.match(r'^/{}/(.+)$'.format(dsid), bfile)
|
|
1181
1278
|
if ms:
|
|
1182
1279
|
qfile = bfile
|
|
@@ -1185,8 +1282,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
1185
1282
|
qfile = "/{}/{}".format(dsid, bfile)
|
|
1186
1283
|
return (bfile, qfile)
|
|
1187
1284
|
|
|
1188
|
-
# archive a backup file for given wfiles/sfiles
|
|
1189
1285
|
def archive_backup_file(self):
|
|
1286
|
+
"""
|
|
1287
|
+
Archive web or saved files as a Quasar backup tar file.
|
|
1288
|
+
|
|
1289
|
+
Bundles the specified web files (params['WF']) or saved files
|
|
1290
|
+
(params['SF']) into a tar archive and transfers it to Quasar storage.
|
|
1291
|
+
Registers the backup record in the bfile RDADB table.
|
|
1292
|
+
Sets self.RETSTAT on failure.
|
|
1293
|
+
"""
|
|
1190
1294
|
tname = 'bfile'
|
|
1191
1295
|
endpoint = self.PGLOG['BACKUPEP']
|
|
1192
1296
|
drpoint = self.PGLOG['DRDATAEP']
|
|
@@ -1292,8 +1396,19 @@ class DsArch(PgArch, PgMeta):
|
|
|
1292
1396
|
self.reset_rdadb_version(dsid)
|
|
1293
1397
|
self.record_delete_directory(None, (self.params['DD'] if 'DD' in self.params else 0))
|
|
1294
1398
|
|
|
1295
|
-
# tarring saved files to Quasar backup file
|
|
1296
1399
|
def tar_backup_savedfiles(self, tarfile, tinfo, ccnt, chkstat):
|
|
1400
|
+
"""
|
|
1401
|
+
Append saved files to a Quasar backup tar archive.
|
|
1402
|
+
|
|
1403
|
+
Args:
|
|
1404
|
+
tarfile -- path to the tar archive file being built
|
|
1405
|
+
tinfo -- dict with tar file metadata (size, checksum, member list)
|
|
1406
|
+
ccnt -- current member file count in the archive
|
|
1407
|
+
chkstat -- dscheck status record for progress tracking
|
|
1408
|
+
|
|
1409
|
+
Returns:
|
|
1410
|
+
Updated member count after adding files.
|
|
1411
|
+
"""
|
|
1297
1412
|
scnt = len(self.params['SF'])
|
|
1298
1413
|
dsid = self.params['DS']
|
|
1299
1414
|
fcnd = "dsid = '{}' and sfile = ".format(dsid)
|
|
@@ -1379,8 +1494,19 @@ class DsArch(PgArch, PgMeta):
|
|
|
1379
1494
|
tinfo['sids'] = sids
|
|
1380
1495
|
return scnt
|
|
1381
1496
|
|
|
1382
|
-
# tarring web files to Quasar backup file
|
|
1383
1497
|
def tar_backup_webfiles(self, tarfile, tinfo, ccnt, chkstat):
|
|
1498
|
+
"""
|
|
1499
|
+
Append web files to a Quasar backup tar archive.
|
|
1500
|
+
|
|
1501
|
+
Args:
|
|
1502
|
+
tarfile -- path to the tar archive file being built
|
|
1503
|
+
tinfo -- dict with tar file metadata (size, checksum, member list)
|
|
1504
|
+
ccnt -- current member file count in the archive
|
|
1505
|
+
chkstat -- dscheck status record for progress tracking
|
|
1506
|
+
|
|
1507
|
+
Returns:
|
|
1508
|
+
Updated member count after adding files.
|
|
1509
|
+
"""
|
|
1384
1510
|
wcnt = len(self.params['WF'])
|
|
1385
1511
|
dsid = self.params['DS']
|
|
1386
1512
|
fcnd = "wfile = "
|
|
@@ -1460,8 +1586,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
1460
1586
|
tinfo['wids'] = wids
|
|
1461
1587
|
return wcnt
|
|
1462
1588
|
|
|
1463
|
-
# retrieve backup tar files from Quasar servers
|
|
1464
1589
|
def retrieve_backup_files(self):
|
|
1590
|
+
"""
|
|
1591
|
+
Download Quasar backup tar files to a local staging directory.
|
|
1592
|
+
|
|
1593
|
+
Fetches each backup file listed in params['QF'] from the Quasar endpoint
|
|
1594
|
+
and saves it locally for subsequent restore operations.
|
|
1595
|
+
Sets self.RETSTAT on transfer failures.
|
|
1596
|
+
"""
|
|
1465
1597
|
tname = 'bfile'
|
|
1466
1598
|
endpoint = self.PGLOG['BACKUPEP']
|
|
1467
1599
|
dsid = self.params['DS']
|
|
@@ -1515,8 +1647,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
1515
1647
|
self.pglog("{} of Quasar Backup file{} downloaded for {}".format(dcnt, self.ALLCNT, s, dsid), self.LOGWRN)
|
|
1516
1648
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1517
1649
|
|
|
1518
|
-
# untar backup file to retrieve web files
|
|
1519
1650
|
def restore_backup_webfiles(self):
|
|
1651
|
+
"""
|
|
1652
|
+
Extract web files from a retrieved Quasar backup tar archive.
|
|
1653
|
+
|
|
1654
|
+
Untars each backup archive and copies the extracted web files to their
|
|
1655
|
+
original GLADE/object-store locations, then updates RDADB records.
|
|
1656
|
+
Sets self.RETSTAT on failure.
|
|
1657
|
+
"""
|
|
1520
1658
|
tname = 'wfile'
|
|
1521
1659
|
dsid = self.params['DS']
|
|
1522
1660
|
bucket = self.PGLOG['OBJCTBKT'] # default object store bucket
|
|
@@ -1585,8 +1723,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
1585
1723
|
self.pglog("{} of {} Object file{} restored for {}".format(ocnt, self.ALLCNT, s, dsid), self.PGOPT['emllog'])
|
|
1586
1724
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1587
1725
|
|
|
1588
|
-
# untar backup tarfile to get a single member file
|
|
1589
1726
|
def get_backup_member_file(self, pgrec, tarfile, tardir):
|
|
1727
|
+
"""
|
|
1728
|
+
Extract a single member file from a backup tar archive.
|
|
1729
|
+
|
|
1730
|
+
Args:
|
|
1731
|
+
pgrec -- RDADB record dict for the file to extract
|
|
1732
|
+
tarfile -- path to the local tar archive
|
|
1733
|
+
tardir -- directory to extract the member file into
|
|
1734
|
+
|
|
1735
|
+
Returns:
|
|
1736
|
+
Path to the extracted file, or None on failure.
|
|
1737
|
+
"""
|
|
1590
1738
|
mfile = wfile = pgrec['wfile']
|
|
1591
1739
|
note = pgrec['note']
|
|
1592
1740
|
while not re.search(r'{}<:>'.format(mfile), note):
|
|
@@ -1603,8 +1751,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
1603
1751
|
ainfo = self.check_local_file(nfile, 0, self.PGOPT['extlog']|self.PFSIZE)
|
|
1604
1752
|
return ainfo
|
|
1605
1753
|
|
|
1606
|
-
# untar backup file to retrieve saved files
|
|
1607
1754
|
def restore_backup_savedfiles(self):
|
|
1755
|
+
"""
|
|
1756
|
+
Extract saved files from a retrieved Quasar backup tar archive.
|
|
1757
|
+
|
|
1758
|
+
Untars each backup archive and copies the extracted saved files back to
|
|
1759
|
+
their original GLADE locations, then updates RDADB records.
|
|
1760
|
+
Sets self.RETSTAT on failure.
|
|
1761
|
+
"""
|
|
1608
1762
|
tname = 'sfile'
|
|
1609
1763
|
dsid = self.params['DS']
|
|
1610
1764
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1673,8 +1827,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
1673
1827
|
self.set_dscheck_dcount(self.ALLCNT, chksize, self.PGOPT['extlog'])
|
|
1674
1828
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1675
1829
|
|
|
1676
|
-
# cross copy quasar backup files between Globus endpoints gdex-quasar and gdex-quasar-drdata
|
|
1677
1830
|
def crosscopy_backup_files(self):
|
|
1831
|
+
"""
|
|
1832
|
+
Cross-copy Quasar backup files between Globus endpoints.
|
|
1833
|
+
|
|
1834
|
+
Transfers backup files between gdex-quasar and gdex-quasar-drdata
|
|
1835
|
+
endpoints using Globus transfer, updating RDADB records on completion.
|
|
1836
|
+
Sets self.RETSTAT on transfer failures.
|
|
1837
|
+
"""
|
|
1678
1838
|
tname = 'bfile'
|
|
1679
1839
|
qtype = 'D'
|
|
1680
1840
|
dsid = self.params['DS']
|
|
@@ -1793,8 +1953,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
1793
1953
|
self.reset_rdadb_version(dsid)
|
|
1794
1954
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
1795
1955
|
|
|
1796
|
-
# validate a webfile archive, fatal is error
|
|
1797
1956
|
def validate_gladearch(self, file, ofile, i):
|
|
1957
|
+
"""
|
|
1958
|
+
Verify that a GLADE archive file exists and matches the expected file info.
|
|
1959
|
+
|
|
1960
|
+
Logs a fatal error when validation fails, setting the file entry at
|
|
1961
|
+
index *i* to None so it is skipped in subsequent processing.
|
|
1962
|
+
|
|
1963
|
+
Args:
|
|
1964
|
+
file -- original local file name (for error messages)
|
|
1965
|
+
ofile -- expected absolute GLADE archive path to check
|
|
1966
|
+
i -- file index (used to null out params entries on failure)
|
|
1967
|
+
"""
|
|
1798
1968
|
info = self.check_local_file(file, 6, self.PGOPT['errlog']|self.PFSIZE)
|
|
1799
1969
|
if not info:
|
|
1800
1970
|
self.pglog("Error archiving {} to {}".format(ofile, file), self.LGEREX)
|
|
@@ -1803,24 +1973,48 @@ class DsArch(PgArch, PgMeta):
|
|
|
1803
1973
|
else:
|
|
1804
1974
|
self.set_local_mode(file, 1, self.PGLOG['FILEMODE'], info['mode'], info['logname'], self.PGOPT['errlog'])
|
|
1805
1975
|
|
|
1806
|
-
# validate an object store file archive, fatal is error
|
|
1807
1976
|
def validate_objectarch(self, file, ofile, bucket, i):
|
|
1977
|
+
"""
|
|
1978
|
+
Verify that an object-store file exists and matches the expected file info.
|
|
1979
|
+
|
|
1980
|
+
Args:
|
|
1981
|
+
file -- original local file name (for error messages)
|
|
1982
|
+
ofile -- object-store key path to check
|
|
1983
|
+
bucket -- object-store bucket name
|
|
1984
|
+
i -- file index (used to null out params entries on failure)
|
|
1985
|
+
"""
|
|
1808
1986
|
info = self.check_object_file(file, bucket, 0, self.PGOPT['errlog'])
|
|
1809
1987
|
if not info:
|
|
1810
1988
|
self.pglog("Error archiving {} to {}-{}".format(ofile, bucket, file), self.LGEREX)
|
|
1811
1989
|
elif self.OPTS['SZ'][2]&2 and self.params['SZ'] and info['data_size'] != self.params['SZ'][i]:
|
|
1812
1990
|
self.pglog("Wrong Sizes: ({}-{}){}/({}){}".format(bucket, file, info['data_size'], ofile, self.params['SZ'][i]), self.LGEREX)
|
|
1813
1991
|
|
|
1814
|
-
# validate an object store file archive, fatal is error
|
|
1815
1992
|
def validate_backarch(self, file, ofile, endpoint, i):
|
|
1993
|
+
"""
|
|
1994
|
+
Verify that a Quasar backup file exists at the given endpoint.
|
|
1995
|
+
|
|
1996
|
+
Args:
|
|
1997
|
+
file -- original local file name (for error messages)
|
|
1998
|
+
ofile -- backup file path on the Quasar endpoint
|
|
1999
|
+
endpoint -- Globus endpoint identifier string
|
|
2000
|
+
i -- file index (used to null out params entries on failure)
|
|
2001
|
+
"""
|
|
1816
2002
|
info = self.check_backup_file(file, endpoint, 0, self.PGOPT['errlog'])
|
|
1817
2003
|
if not info:
|
|
1818
2004
|
self.pglog("Error archiving {} to {}-{}".format(ofile, endpoint, file), self.LGEREX)
|
|
1819
2005
|
elif self.OPTS['SZ'][2]&2 and self.params['SZ'] and info['data_size'] != self.params['SZ'][i]:
|
|
1820
2006
|
self.pglog("Wrong Sizes: ({}-{}){}/({}){}".format(endpoint, file, info['data_size'], ofile, self.params['SZ'][i]), self.LGEREX)
|
|
1821
2007
|
|
|
1822
|
-
# get dataset info
|
|
1823
2008
|
def get_dataset_info(self):
|
|
2009
|
+
"""
|
|
2010
|
+
Retrieve and display dataset metadata from RDADB.
|
|
2011
|
+
|
|
2012
|
+
Fetches the dataset record (and optionally period info) for the dataset
|
|
2013
|
+
specified in params['DS'] and writes formatted output to self.OUTPUT.
|
|
2014
|
+
|
|
2015
|
+
Returns:
|
|
2016
|
+
The dataset record dict, or None if not found.
|
|
2017
|
+
"""
|
|
1824
2018
|
tname = 'dataset'
|
|
1825
2019
|
dsid = self.params['DS']
|
|
1826
2020
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1841,8 +2035,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
1841
2035
|
if 'WN' in self.params: self.view_filenumber(dsid, 0)
|
|
1842
2036
|
return 1 # get one dataset record
|
|
1843
2037
|
|
|
1844
|
-
# get dataset period information
|
|
1845
2038
|
def get_period_info(self):
|
|
2039
|
+
"""
|
|
2040
|
+
Retrieve and display dataset time-period records from RDADB.
|
|
2041
|
+
|
|
2042
|
+
Queries the dsperiod table for the current dataset and writes formatted
|
|
2043
|
+
period information to self.OUTPUT.
|
|
2044
|
+
"""
|
|
1846
2045
|
tname = "dsperiod"
|
|
1847
2046
|
dsid = self.params['DS']
|
|
1848
2047
|
hash = self.TBLHASH[tname]
|
|
@@ -1857,8 +2056,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
1857
2056
|
s = 's' if cnt > 1 else ''
|
|
1858
2057
|
self.pglog("{} period record{} retrieved for {}".format(cnt, s, dsid), self.LOGWRN)
|
|
1859
2058
|
|
|
1860
|
-
|
|
1861
|
-
|
|
2059
|
+
def set_dataset_info(self, include=None):
|
|
2060
|
+
"""
|
|
2061
|
+
Insert or update the dataset record in RDADB.
|
|
2062
|
+
|
|
2063
|
+
Reads dataset fields from params, builds an update/insert record, and
|
|
2064
|
+
calls the appropriate database function. When *include* is 'P', only
|
|
2065
|
+
period-related fields are updated.
|
|
2066
|
+
|
|
2067
|
+
Args:
|
|
2068
|
+
include -- field subset selector string, or None for all fields
|
|
2069
|
+
"""
|
|
1862
2070
|
tname = 'dataset'
|
|
1863
2071
|
dsid = self.params['DS']
|
|
1864
2072
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1895,8 +2103,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
1895
2103
|
if pcnt + mcnt:
|
|
1896
2104
|
self.reset_rdadb_version(dsid)
|
|
1897
2105
|
|
|
1898
|
-
# add new or modify existing period record into RDADB
|
|
1899
2106
|
def set_period_info(self, dcnd):
|
|
2107
|
+
"""
|
|
2108
|
+
Insert or update dataset time-period records in RDADB via the sdp command.
|
|
2109
|
+
|
|
2110
|
+
Args:
|
|
2111
|
+
dcnd -- SQL condition string identifying the dataset (e.g. "dsid='d123456'")
|
|
2112
|
+
"""
|
|
1900
2113
|
tname = 'dsperiod'
|
|
1901
2114
|
dsid = self.params['DS']
|
|
1902
2115
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1934,8 +2147,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
1934
2147
|
if pcnt > 0: self.pglog("{} of {} period{} modified for {}!".format(pcnt, allcnt, s, dsid), self.LOGWRN)
|
|
1935
2148
|
return pcnt
|
|
1936
2149
|
|
|
1937
|
-
# delete group information for given dataset and index list
|
|
1938
2150
|
def delete_group_info(self):
|
|
2151
|
+
"""
|
|
2152
|
+
Delete group records from RDADB for the current dataset.
|
|
2153
|
+
|
|
2154
|
+
Verifies that no dependent file records (web, saved, help) remain before
|
|
2155
|
+
deleting each group index in params['GI']. Logs an error and skips any
|
|
2156
|
+
group that still has associated files.
|
|
2157
|
+
"""
|
|
1939
2158
|
tname = 'dsgroup'
|
|
1940
2159
|
dsid = self.params['DS']
|
|
1941
2160
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -1961,8 +2180,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
1961
2180
|
self.pglog("{} of {} group{} deleted from {}".format(delcnt, self.ALLCNT, s, dsid), self.LOGWRN)
|
|
1962
2181
|
if delcnt > 0: self.reset_rdadb_version(dsid)
|
|
1963
2182
|
|
|
1964
|
-
# get group information
|
|
1965
2183
|
def get_group_info(self):
|
|
2184
|
+
"""
|
|
2185
|
+
Retrieve and display dataset group records from RDADB.
|
|
2186
|
+
|
|
2187
|
+
Fetches dsgroup rows for the current dataset (filtered by any -GI/-GN
|
|
2188
|
+
params) and writes formatted output to self.OUTPUT.
|
|
2189
|
+
"""
|
|
1966
2190
|
tname = "dsgroup"
|
|
1967
2191
|
dsid = self.params['DS']
|
|
1968
2192
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -2000,8 +2224,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
2000
2224
|
else:
|
|
2001
2225
|
self.pglog("No group found for " + condition, self.LOGWRN)
|
|
2002
2226
|
|
|
2003
|
-
# add or modify group information
|
|
2004
2227
|
def set_group_info(self):
|
|
2228
|
+
"""
|
|
2229
|
+
Insert or update group records in RDADB for the current dataset.
|
|
2230
|
+
|
|
2231
|
+
Iterates over params['GI'], building insert/update records from params
|
|
2232
|
+
and calling the appropriate database function for each group index.
|
|
2233
|
+
Resets the RDADB version after changes.
|
|
2234
|
+
"""
|
|
2005
2235
|
if self.ALLCNT == 0 or self.params['GI'][0] == 0: return # skip group index 0
|
|
2006
2236
|
tname = 'dsgroup'
|
|
2007
2237
|
dsid = self.params['DS']
|
|
@@ -2053,8 +2283,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
2053
2283
|
self.pglog("{}/{} of {} group{} added/modified for {}!".format(addcnt, modcnt, self.ALLCNT, s, dsid), self.LOGWRN)
|
|
2054
2284
|
if (addcnt + modcnt + tcnt) > 0: self.reset_rdadb_version(dsid)
|
|
2055
2285
|
|
|
2056
|
-
# set subgroups and files to Internal for given group index
|
|
2057
2286
|
def set_for_internal_group(self, gindex):
|
|
2287
|
+
"""
|
|
2288
|
+
Mark a group and all its descendant files as Internal ('I').
|
|
2289
|
+
|
|
2290
|
+
Recursively updates dsgroup records and the status column of all
|
|
2291
|
+
associated web/saved files to 'I', adjusting on-disk file permissions
|
|
2292
|
+
via change_wfile_mode() where needed.
|
|
2293
|
+
|
|
2294
|
+
Args:
|
|
2295
|
+
gindex -- group index to make internal
|
|
2296
|
+
"""
|
|
2058
2297
|
tname = 'dsgroup'
|
|
2059
2298
|
dsid = self.params['DS']
|
|
2060
2299
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -2085,8 +2324,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2085
2324
|
for i in range(cnt):
|
|
2086
2325
|
self.set_for_internal_group(pgrecs['gindex'][i])
|
|
2087
2326
|
|
|
2088
|
-
# get WEB file information
|
|
2089
2327
|
def get_webfile_info(self):
|
|
2328
|
+
"""
|
|
2329
|
+
Retrieve and display web file records from RDADB.
|
|
2330
|
+
|
|
2331
|
+
Queries the wfile table for the current dataset (applying any filter
|
|
2332
|
+
params) and writes formatted output to self.OUTPUT.
|
|
2333
|
+
"""
|
|
2090
2334
|
tname = "wfile"
|
|
2091
2335
|
dsid = self.params['DS']
|
|
2092
2336
|
hash = self.TBLHASH[tname]
|
|
@@ -2137,8 +2381,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2137
2381
|
else:
|
|
2138
2382
|
self.pglog("No Web file found for " + condition, self.LOGWRN)
|
|
2139
2383
|
|
|
2140
|
-
# get Help file information
|
|
2141
2384
|
def get_helpfile_info(self):
|
|
2385
|
+
"""
|
|
2386
|
+
Retrieve and display help file records from RDADB.
|
|
2387
|
+
|
|
2388
|
+
Queries the hfile table for the current dataset (applying any filter
|
|
2389
|
+
params) and writes formatted output to self.OUTPUT.
|
|
2390
|
+
"""
|
|
2142
2391
|
tjoin = tname = "hfile"
|
|
2143
2392
|
dsid = self.params['DS']
|
|
2144
2393
|
hash = self.TBLHASH[tname]
|
|
@@ -2172,8 +2421,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2172
2421
|
else:
|
|
2173
2422
|
self.pglog("No Help file found for " + condition, self.LOGWRN)
|
|
2174
2423
|
|
|
2175
|
-
# get Saved file information
|
|
2176
2424
|
def get_savedfile_info(self):
|
|
2425
|
+
"""
|
|
2426
|
+
Retrieve and display saved file records from RDADB.
|
|
2427
|
+
|
|
2428
|
+
Queries the sfile table for the current dataset (applying any filter
|
|
2429
|
+
params) and writes formatted output to self.OUTPUT.
|
|
2430
|
+
"""
|
|
2177
2431
|
tjoin = tname = "sfile"
|
|
2178
2432
|
dojoin = 0
|
|
2179
2433
|
dsid = self.params['DS']
|
|
@@ -2221,8 +2475,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2221
2475
|
else:
|
|
2222
2476
|
self.pglog("No Saved file found for " + condition, self.LOGWRN)
|
|
2223
2477
|
|
|
2224
|
-
# get Quasar Backup file information
|
|
2225
2478
|
def get_backfile_info(self):
|
|
2479
|
+
"""
|
|
2480
|
+
Retrieve and display Quasar backup file records from RDADB.
|
|
2481
|
+
|
|
2482
|
+
Queries the bfile table for the current dataset and writes formatted
|
|
2483
|
+
output to self.OUTPUT.
|
|
2484
|
+
"""
|
|
2226
2485
|
tname = "bfile"
|
|
2227
2486
|
dsid = self.params['DS']
|
|
2228
2487
|
hash = self.TBLHASH[tname]
|
|
@@ -2256,8 +2515,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
2256
2515
|
else:
|
|
2257
2516
|
self.pglog("No Quasar Backup file found for " + condition, self.LOGWRN)
|
|
2258
2517
|
|
|
2259
|
-
|
|
2260
|
-
|
|
2518
|
+
def set_webfile_info(self, include=None):
|
|
2519
|
+
"""
|
|
2520
|
+
Insert or update web file records in RDADB for the current dataset.
|
|
2521
|
+
|
|
2522
|
+
Iterates over params['WF'], calling set_one_webfile() for each entry.
|
|
2523
|
+
Handles display-order management, metadata link updates, and optional
|
|
2524
|
+
metadata XML gathering. Resets the RDADB version after changes.
|
|
2525
|
+
|
|
2526
|
+
Args:
|
|
2527
|
+
include -- optional field subset selector string passed to get_condition()
|
|
2528
|
+
"""
|
|
2261
2529
|
bidx = 0
|
|
2262
2530
|
dftloc = None
|
|
2263
2531
|
tname = 'wfile'
|
|
@@ -2352,8 +2620,23 @@ class DsArch(PgArch, PgMeta):
|
|
|
2352
2620
|
self.reset_rdadb_version(dsid)
|
|
2353
2621
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
2354
2622
|
|
|
2355
|
-
|
|
2356
|
-
|
|
2623
|
+
def set_one_webfile(self, i, pgrec, file, flds, type, info=None, ndsid=None, sact=0):
|
|
2624
|
+
"""
|
|
2625
|
+
Insert or update a single web file record in RDADB.
|
|
2626
|
+
|
|
2627
|
+
Args:
|
|
2628
|
+
i -- file index into params arrays
|
|
2629
|
+
pgrec -- existing wfile record dict, or None for a new record
|
|
2630
|
+
file -- web file path (relative to dataset root)
|
|
2631
|
+
flds -- list of field names to set (from get_field_keys)
|
|
2632
|
+
type -- web file type character
|
|
2633
|
+
info -- file metadata dict (size, date, checksum) from check_local_file
|
|
2634
|
+
ndsid -- target dataset ID when moving between datasets; None = current
|
|
2635
|
+
sact -- non-zero when called from set_webfile_info (suppress some side-effects)
|
|
2636
|
+
|
|
2637
|
+
Returns:
|
|
2638
|
+
The wid (web file ID) of the inserted/updated record, or None on failure.
|
|
2639
|
+
"""
|
|
2357
2640
|
tname = 'wfile'
|
|
2358
2641
|
gindex = (self.params['GI'][i] if 'GI' in self.params and self.OPTS['GI'][2]&2 == 0 else (pgrec['gindex'] if pgrec else 0))
|
|
2359
2642
|
dsid = ndsid if ndsid else self.params['DS']
|
|
@@ -2438,8 +2721,19 @@ class DsArch(PgArch, PgMeta):
|
|
|
2438
2721
|
self.save_filenumber(None, sact, 1, fcnt)
|
|
2439
2722
|
return wid
|
|
2440
2723
|
|
|
2441
|
-
|
|
2442
|
-
|
|
2724
|
+
def check_file_flag(self, file, info, record, pgrec=None):
|
|
2725
|
+
"""
|
|
2726
|
+
Determine and set the file flag ('F' for full, 'P' for partial) in a record.
|
|
2727
|
+
|
|
2728
|
+
Compares the expected file size from *info* (or the archived file) against
|
|
2729
|
+
a minimum-size threshold and updates record['fileflag'] accordingly.
|
|
2730
|
+
|
|
2731
|
+
Args:
|
|
2732
|
+
file -- file name string (for logging)
|
|
2733
|
+
info -- file metadata dict with at least a 'data_size' key
|
|
2734
|
+
record -- mutable record dict to update with the derived fileflag
|
|
2735
|
+
pgrec -- existing database record dict for comparison, or None
|
|
2736
|
+
"""
|
|
2443
2737
|
if not info: return self.SUCCESS
|
|
2444
2738
|
fflag = 'F' if info['isfile'] else 'P'
|
|
2445
2739
|
if 'fileflag' in record:
|
|
@@ -2449,8 +2743,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2449
2743
|
record['fileflag'] = fflag
|
|
2450
2744
|
return self.SUCCESS
|
|
2451
2745
|
|
|
2452
|
-
# add or modify Help file information
|
|
2453
2746
|
def set_helpfile_info(self):
|
|
2747
|
+
"""
|
|
2748
|
+
Insert or update help file records in RDADB for the current dataset.
|
|
2749
|
+
|
|
2750
|
+
Iterates over params['HF'], calling set_one_helpfile() for each entry.
|
|
2751
|
+
Handles display-order management and resets the RDADB version after changes.
|
|
2752
|
+
"""
|
|
2454
2753
|
bidx = 0
|
|
2455
2754
|
tname = 'hfile'
|
|
2456
2755
|
dsid = self.params['DS']
|
|
@@ -2503,8 +2802,22 @@ class DsArch(PgArch, PgMeta):
|
|
|
2503
2802
|
if (self.ADDCNT + self.MODCNT + reorder) > 0: self.reset_rdadb_version(dsid)
|
|
2504
2803
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
2505
2804
|
|
|
2506
|
-
|
|
2507
|
-
|
|
2805
|
+
def set_one_helpfile(self, i, pgrec, file, flds, type, info=None, ndsid=None):
|
|
2806
|
+
"""
|
|
2807
|
+
Insert or update a single help file record in RDADB.
|
|
2808
|
+
|
|
2809
|
+
Args:
|
|
2810
|
+
i -- file index into params arrays
|
|
2811
|
+
pgrec -- existing hfile record dict, or None for a new record
|
|
2812
|
+
file -- help file path (relative to dataset help root)
|
|
2813
|
+
flds -- list of field names to set
|
|
2814
|
+
type -- help file type character
|
|
2815
|
+
info -- file metadata dict (size, date, checksum), or None
|
|
2816
|
+
ndsid -- target dataset ID when moving between datasets; None = current
|
|
2817
|
+
|
|
2818
|
+
Returns:
|
|
2819
|
+
The hid (help file ID) of the inserted/updated record, or None on failure.
|
|
2820
|
+
"""
|
|
2508
2821
|
tname = 'hfile'
|
|
2509
2822
|
dsid = ndsid if ndsid else self.params['DS']
|
|
2510
2823
|
hid = pgrec['hid'] if pgrec else 0
|
|
@@ -2575,8 +2888,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2575
2888
|
fcnt = 1
|
|
2576
2889
|
return hid
|
|
2577
2890
|
|
|
2578
|
-
# add or modify save file information
|
|
2579
2891
|
def set_savedfile_info(self):
|
|
2892
|
+
"""
|
|
2893
|
+
Insert or update saved file records in RDADB for the current dataset.
|
|
2894
|
+
|
|
2895
|
+
Iterates over params['SF'], calling set_one_savedfile() for each entry.
|
|
2896
|
+
Handles display-order management and resets the RDADB version after changes.
|
|
2897
|
+
"""
|
|
2580
2898
|
bidx = 0
|
|
2581
2899
|
dftloc = None
|
|
2582
2900
|
tname = 'sfile'
|
|
@@ -2659,8 +2977,23 @@ class DsArch(PgArch, PgMeta):
|
|
|
2659
2977
|
self.reset_rdadb_version(dsid)
|
|
2660
2978
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
2661
2979
|
|
|
2662
|
-
|
|
2663
|
-
|
|
2980
|
+
def set_one_savedfile(self, i, pgrec, file, flds, type, info=None, ndsid=None, sact=0):
|
|
2981
|
+
"""
|
|
2982
|
+
Insert or update a single saved file record in RDADB.
|
|
2983
|
+
|
|
2984
|
+
Args:
|
|
2985
|
+
i -- file index into params arrays
|
|
2986
|
+
pgrec -- existing sfile record dict, or None for a new record
|
|
2987
|
+
file -- saved file path (relative to dataset saved root)
|
|
2988
|
+
flds -- list of field names to set
|
|
2989
|
+
type -- saved file type character
|
|
2990
|
+
info -- file metadata dict (size, date, checksum), or None
|
|
2991
|
+
ndsid -- target dataset ID when moving between datasets; None = current
|
|
2992
|
+
sact -- non-zero when called from set_savedfile_info (suppress side-effects)
|
|
2993
|
+
|
|
2994
|
+
Returns:
|
|
2995
|
+
The sid (saved file ID) of the inserted/updated record, or None on failure.
|
|
2996
|
+
"""
|
|
2664
2997
|
tname = 'sfile'
|
|
2665
2998
|
gindex = (self.params['GI'][i] if 'GI' in self.params and self.OPTS['GI'][2]&2 == 0 else (pgrec['gindex'] if pgrec else 0))
|
|
2666
2999
|
dsid = ndsid if ndsid else self.params['DS']
|
|
@@ -2745,8 +3078,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2745
3078
|
self.save_filenumber(None, sact, 1, fcnt)
|
|
2746
3079
|
return sid
|
|
2747
3080
|
|
|
2748
|
-
# add or modify Quasar backup file information
|
|
2749
3081
|
def set_backfile_info(self):
|
|
3082
|
+
"""
|
|
3083
|
+
Insert or update Quasar backup file records in RDADB for the current dataset.
|
|
3084
|
+
|
|
3085
|
+
Iterates over params['QF'], calling set_one_backfile() for each entry
|
|
3086
|
+
and resetting the RDADB version after changes.
|
|
3087
|
+
"""
|
|
2750
3088
|
bidx = zipped = 0
|
|
2751
3089
|
tname = 'bfile'
|
|
2752
3090
|
dsid = self.params['DS']
|
|
@@ -2799,8 +3137,22 @@ class DsArch(PgArch, PgMeta):
|
|
|
2799
3137
|
self.reset_rdadb_version(dsid)
|
|
2800
3138
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
2801
3139
|
|
|
2802
|
-
|
|
2803
|
-
|
|
3140
|
+
def set_one_backfile(self, i, pgrec, file, flds, type, ndsid=None, record=None):
|
|
3141
|
+
"""
|
|
3142
|
+
Insert or update a single Quasar backup file record in RDADB.
|
|
3143
|
+
|
|
3144
|
+
Args:
|
|
3145
|
+
i -- file index into params arrays
|
|
3146
|
+
pgrec -- existing bfile record dict, or None for a new record
|
|
3147
|
+
file -- backup file name
|
|
3148
|
+
flds -- list of field names to set
|
|
3149
|
+
type -- backup file type character
|
|
3150
|
+
ndsid -- target dataset ID when moving between datasets; None = current
|
|
3151
|
+
record -- pre-built record dict to use instead of building from params
|
|
3152
|
+
|
|
3153
|
+
Returns:
|
|
3154
|
+
The bid (backup file ID) of the inserted/updated record, or None on failure.
|
|
3155
|
+
"""
|
|
2804
3156
|
tname = 'bfile'
|
|
2805
3157
|
dsid = ndsid if ndsid else self.params['DS']
|
|
2806
3158
|
bid = pgrec['bid'] if pgrec else 0
|
|
@@ -2837,8 +3189,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
2837
3189
|
self.ADDCNT += 1
|
|
2838
3190
|
return bid
|
|
2839
3191
|
|
|
2840
|
-
# moving WEB files tochange file paths/names, and/or from one dataset to another
|
|
2841
3192
|
def move_web_files(self):
|
|
3193
|
+
"""
|
|
3194
|
+
Rename or relocate web files on disk and update their RDADB records.
|
|
3195
|
+
|
|
3196
|
+
Handles renaming within the same dataset, cross-dataset moves, and
|
|
3197
|
+
type changes (e.g. D→N). Supports -TS (to-saved) conversions and
|
|
3198
|
+
updates backup file notes via save_move_info().
|
|
3199
|
+
Sets self.RETSTAT on unresolved failures.
|
|
3200
|
+
"""
|
|
2842
3201
|
tname = 'wfile'
|
|
2843
3202
|
dsid = self.params['DS']
|
|
2844
3203
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -2981,8 +3340,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
2981
3340
|
self.reset_rdadb_version(self.params['OD'])
|
|
2982
3341
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
2983
3342
|
|
|
2984
|
-
# moving Help files to change file paths/names, and/or from one dataset to another
|
|
2985
3343
|
def move_help_files(self):
|
|
3344
|
+
"""
|
|
3345
|
+
Rename or relocate help files on disk and update their RDADB records.
|
|
3346
|
+
|
|
3347
|
+
Handles renaming within the same dataset and cross-dataset moves, along
|
|
3348
|
+
with any type changes. Sets self.RETSTAT on unresolved failures.
|
|
3349
|
+
"""
|
|
2986
3350
|
tname = 'hfile'
|
|
2987
3351
|
dsid = self.params['DS']
|
|
2988
3352
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -3086,8 +3450,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
3086
3450
|
self.reset_rdadb_version(self.params['OD'])
|
|
3087
3451
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ''
|
|
3088
3452
|
|
|
3089
|
-
# moving Saved files to Web files, both on glade and object store
|
|
3090
3453
|
def saved_to_web_files(self):
|
|
3454
|
+
"""
|
|
3455
|
+
Convert saved files to web files (-TS action).
|
|
3456
|
+
|
|
3457
|
+
Copies each saved file to its web file destination (GLADE and/or object
|
|
3458
|
+
store), creates a new wfile RDADB record, and removes the old sfile
|
|
3459
|
+
record. Preserves backup links by updating bfile.note via save_move_info().
|
|
3460
|
+
Sets self.RETSTAT on unresolved failures.
|
|
3461
|
+
"""
|
|
3091
3462
|
tname = 'wfile'
|
|
3092
3463
|
dsid = self.params['DS']
|
|
3093
3464
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -3248,8 +3619,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
3248
3619
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3249
3620
|
if dslocflags: self.set_dataset_locflag(dsid, dslocflags.pop())
|
|
3250
3621
|
|
|
3251
|
-
# delete Web files from a given dataset
|
|
3252
3622
|
def delete_web_files(self):
|
|
3623
|
+
"""
|
|
3624
|
+
Delete web files from GLADE/object store and remove their RDADB records.
|
|
3625
|
+
|
|
3626
|
+
Iterates over params['WF'], removes each file from disk and/or the
|
|
3627
|
+
object store, then deletes the wfile RDADB row. Optionally triggers
|
|
3628
|
+
metadata XML deletion (-DX). Sets self.RETSTAT on failures.
|
|
3629
|
+
"""
|
|
3253
3630
|
tname = 'wfile'
|
|
3254
3631
|
dsid = self.params['DS']
|
|
3255
3632
|
bucket = self.PGLOG['OBJCTBKT']
|
|
@@ -3350,8 +3727,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
3350
3727
|
self.reset_rdadb_version(dsid)
|
|
3351
3728
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3352
3729
|
|
|
3353
|
-
# delete Help files from a given dataset
|
|
3354
3730
|
def delete_help_files(self):
|
|
3731
|
+
"""
|
|
3732
|
+
Delete help files from GLADE/object store and remove their RDADB records.
|
|
3733
|
+
|
|
3734
|
+
Iterates over params['HF'], removes each file from disk and/or the
|
|
3735
|
+
object store, then deletes the hfile RDADB row.
|
|
3736
|
+
Sets self.RETSTAT on failures.
|
|
3737
|
+
"""
|
|
3355
3738
|
tname = 'hfile'
|
|
3356
3739
|
dsid = self.params['DS']
|
|
3357
3740
|
bucket = self.PGLOG['OBJCTBKT']
|
|
@@ -3433,8 +3816,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
3433
3816
|
if (dcnt + reorder) > 0: self.reset_rdadb_version(dsid)
|
|
3434
3817
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3435
3818
|
|
|
3436
|
-
# moving Saved files to change file paths/names, and/or from one dataset to another
|
|
3437
3819
|
def move_saved_files(self):
|
|
3820
|
+
"""
|
|
3821
|
+
Rename or relocate saved files on disk and update their RDADB records.
|
|
3822
|
+
|
|
3823
|
+
Handles renaming within the same dataset and cross-dataset moves, along
|
|
3824
|
+
with any type changes. Sets self.RETSTAT on unresolved failures.
|
|
3825
|
+
"""
|
|
3438
3826
|
tname = 'sfile'
|
|
3439
3827
|
dsid = self.params['DS']
|
|
3440
3828
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -3543,8 +3931,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
3543
3931
|
self.reset_rdadb_version(self.params['OD'])
|
|
3544
3932
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3545
3933
|
|
|
3546
|
-
# moving Quasar backup files from one dataset to another
|
|
3547
3934
|
def move_backup_files(self):
|
|
3935
|
+
"""
|
|
3936
|
+
Move Quasar backup files to a different dataset and update RDADB records.
|
|
3937
|
+
|
|
3938
|
+
Transfers ownership of bfile records to the target dataset (-OD) and
|
|
3939
|
+
updates the backup archive note via save_move_info().
|
|
3940
|
+
Sets self.RETSTAT on failures.
|
|
3941
|
+
"""
|
|
3548
3942
|
s = 's' if self.ALLCNT > 1 else ''
|
|
3549
3943
|
tname = 'bfile'
|
|
3550
3944
|
dsid = self.params['DS']
|
|
@@ -3640,8 +4034,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
3640
4034
|
self.reset_rdadb_version(self.params['OD'])
|
|
3641
4035
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3642
4036
|
|
|
3643
|
-
# moving Web files to Saved files, for files both on glade and object store
|
|
3644
4037
|
def web_to_saved_files(self):
|
|
4038
|
+
"""
|
|
4039
|
+
Convert web files to saved files (-TW action).
|
|
4040
|
+
|
|
4041
|
+
Copies each web file to its saved-file destination on GLADE, creates a
|
|
4042
|
+
new sfile RDADB record, and removes the old wfile record. Optionally
|
|
4043
|
+
also deletes the file from the object store. Preserves backup links.
|
|
4044
|
+
Sets self.RETSTAT on unresolved failures.
|
|
4045
|
+
"""
|
|
3645
4046
|
s = 's' if self.ALLCNT > 1 else ''
|
|
3646
4047
|
tname = 'sfile'
|
|
3647
4048
|
dsid = self.params['DS']
|
|
@@ -3792,8 +4193,20 @@ class DsArch(PgArch, PgMeta):
|
|
|
3792
4193
|
self.reset_rdadb_version(self.params['OD'])
|
|
3793
4194
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3794
4195
|
|
|
3795
|
-
# get date/time/size info from given file record
|
|
3796
4196
|
def get_file_origin_info(self, fname, pgrec):
|
|
4197
|
+
"""
|
|
4198
|
+
Extract file metadata (date, time, size) from an existing RDADB record.
|
|
4199
|
+
|
|
4200
|
+
Used to populate the *info* dict expected by set_one_webfile() and
|
|
4201
|
+
similar methods when the local file is no longer present.
|
|
4202
|
+
|
|
4203
|
+
Args:
|
|
4204
|
+
fname -- file name string (for logging)
|
|
4205
|
+
pgrec -- existing database record dict containing date/time/size fields
|
|
4206
|
+
|
|
4207
|
+
Returns:
|
|
4208
|
+
Dict with 'date_modified', 'time_modified', 'data_size' keys, or None.
|
|
4209
|
+
"""
|
|
3797
4210
|
info = {'isfile': (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size': pgrec['data_size']}
|
|
3798
4211
|
info['fname'] = op.basename(fname)
|
|
3799
4212
|
info['date_modified'] = pgrec['date_modified']
|
|
@@ -3802,8 +4215,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
3802
4215
|
info['time_created'] = pgrec['time_created']
|
|
3803
4216
|
return info
|
|
3804
4217
|
|
|
3805
|
-
# delete saved files from a given dataset
|
|
3806
4218
|
def delete_saved_files(self):
|
|
4219
|
+
"""
|
|
4220
|
+
Delete saved files from GLADE and remove their RDADB records.
|
|
4221
|
+
|
|
4222
|
+
Iterates over params['SF'], removes each file from disk, then deletes
|
|
4223
|
+
the sfile RDADB row. Sets self.RETSTAT on failures.
|
|
4224
|
+
"""
|
|
3807
4225
|
tname = 'sfile'
|
|
3808
4226
|
dsid = self.params['DS']
|
|
3809
4227
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -3875,8 +4293,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
3875
4293
|
self.reset_rdadb_version(dsid)
|
|
3876
4294
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3877
4295
|
|
|
3878
|
-
# delete Quasar backup files from a given dataset
|
|
3879
4296
|
def delete_backup_files(self):
|
|
4297
|
+
"""
|
|
4298
|
+
Delete Quasar backup files from Quasar storage and remove RDADB records.
|
|
4299
|
+
|
|
4300
|
+
Iterates over params['QF'], requests deletion from the Quasar endpoint,
|
|
4301
|
+
then removes the bfile RDADB row. Sets self.RETSTAT on failures.
|
|
4302
|
+
"""
|
|
3880
4303
|
tname = 'bfile'
|
|
3881
4304
|
dsid = self.params['DS']
|
|
3882
4305
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -3938,8 +4361,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
3938
4361
|
self.reset_rdadb_version(dsid)
|
|
3939
4362
|
if 'EM' in self.params: self.PGLOG['PRGMSG'] = ""
|
|
3940
4363
|
|
|
3941
|
-
# change existing group indices to new indices.
|
|
3942
4364
|
def change_group_index(self):
|
|
4365
|
+
"""
|
|
4366
|
+
Reassign files from one group index to another in RDADB.
|
|
4367
|
+
|
|
4368
|
+
Maps each source group index (params['OG']) to the corresponding
|
|
4369
|
+
destination index (params['GI']), updates the gindex column of
|
|
4370
|
+
matching wfile/sfile records, and resets file counts for affected groups.
|
|
4371
|
+
"""
|
|
3943
4372
|
tname = 'dsgroup'
|
|
3944
4373
|
dsid = self.params['DS']
|
|
3945
4374
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -4006,8 +4435,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
4006
4435
|
self.pglog("{}/{} associated Saved/Web file record{} modified for new top group".format(tscnt, twcnt, s), self.LOGWRN)
|
|
4007
4436
|
self.reset_rdadb_version(dsid)
|
|
4008
4437
|
|
|
4009
|
-
|
|
4010
|
-
|
|
4438
|
+
def view_keyvalues(self, dsid, kvalues, getkeys=0):
|
|
4439
|
+
"""
|
|
4440
|
+
Display specialist-defined key/value metadata pairs for a dataset.
|
|
4441
|
+
|
|
4442
|
+
Args:
|
|
4443
|
+
dsid -- dataset ID string
|
|
4444
|
+
kvalues -- list of key=value strings or key names to display
|
|
4445
|
+
getkeys -- if non-zero, only list available key names (no values)
|
|
4446
|
+
"""
|
|
4011
4447
|
cond = "dsid = '{}' ".format(dsid)
|
|
4012
4448
|
count = 0
|
|
4013
4449
|
cnt = len(kvalues) if kvalues else 0
|
|
@@ -4037,8 +4473,14 @@ class DsArch(PgArch, PgMeta):
|
|
|
4037
4473
|
self.OUTPUT.write("{}=>{}{}\n".format(values['okey'][i], values['value'][i], self.params['DV']))
|
|
4038
4474
|
return count
|
|
4039
4475
|
|
|
4040
|
-
# set specialist defiend key/value pairs for given dsid
|
|
4041
4476
|
def set_keyvalues(self, dsid, kvalues):
|
|
4477
|
+
"""
|
|
4478
|
+
Insert or update specialist-defined key/value metadata pairs for a dataset.
|
|
4479
|
+
|
|
4480
|
+
Args:
|
|
4481
|
+
dsid -- dataset ID string
|
|
4482
|
+
kvalues -- list of 'key=value' strings to store in RDADB
|
|
4483
|
+
"""
|
|
4042
4484
|
cnt = len(kvalues) if kvalues else 0
|
|
4043
4485
|
if cnt == 0: return 0
|
|
4044
4486
|
s = 's' if cnt > 1 else ''
|
|
@@ -4067,8 +4509,16 @@ class DsArch(PgArch, PgMeta):
|
|
|
4067
4509
|
self.pglog("{}/{}/{} of {} key/value pairs added/modified/deleted for {}!".format(acnt, mcnt, dcnt, cnt, dsid), self.LOGWRN)
|
|
4068
4510
|
return (acnt + mcnt + dcnt)
|
|
4069
4511
|
|
|
4070
|
-
# record moved web file info
|
|
4071
4512
|
def set_web_move(self, pgrec):
|
|
4513
|
+
"""
|
|
4514
|
+
Record a web-file move event in the dataset metadata.
|
|
4515
|
+
|
|
4516
|
+
Appends move information to the relevant metadata record so the
|
|
4517
|
+
history of file relocations is preserved in RDADB.
|
|
4518
|
+
|
|
4519
|
+
Args:
|
|
4520
|
+
pgrec -- existing wfile record dict for the file being moved
|
|
4521
|
+
"""
|
|
4072
4522
|
date = self.curdate()
|
|
4073
4523
|
cond = "wid = {} and date = '{}'".format(pgrec['wid'], date)
|
|
4074
4524
|
if not self.pgget("wmove", "", cond, self.LGWNEX):
|
|
@@ -4076,8 +4526,21 @@ class DsArch(PgArch, PgMeta):
|
|
|
4076
4526
|
'wfile': pgrec['wfile'], 'wid': pgrec['wid'], 'date': date}
|
|
4077
4527
|
self.pgadd("wmove", record, self.LGWNEX)
|
|
4078
4528
|
|
|
4079
|
-
# reset file counts for saved groups
|
|
4080
4529
|
def reset_group_filenumber(self, dsid, act):
|
|
4530
|
+
"""
|
|
4531
|
+
Recompute and update file-count columns for all dataset groups.
|
|
4532
|
+
|
|
4533
|
+
Recalculates webcnt/savedcnt and related size columns from current
|
|
4534
|
+
file records and writes the corrected values back to the dsgroup and
|
|
4535
|
+
dataset tables.
|
|
4536
|
+
|
|
4537
|
+
Args:
|
|
4538
|
+
dsid -- dataset ID string
|
|
4539
|
+
act -- action bitmask controlling which file types to recount
|
|
4540
|
+
|
|
4541
|
+
Returns:
|
|
4542
|
+
Number of records updated.
|
|
4543
|
+
"""
|
|
4081
4544
|
ucnt = 0
|
|
4082
4545
|
gindices = sorted(self.CHGGRPS)
|
|
4083
4546
|
if gindices and gindices[0] != 0:
|
|
@@ -4087,8 +4550,20 @@ class DsArch(PgArch, PgMeta):
|
|
|
4087
4550
|
ucnt += self.reset_filenumber(dsid, 0, act)
|
|
4088
4551
|
return ucnt
|
|
4089
4552
|
|
|
4090
|
-
# reset top group indices for given groups
|
|
4091
4553
|
def reset_top_group_index(self, dsid, act):
|
|
4554
|
+
"""
|
|
4555
|
+
Recalculate and update the top-level group index (tindex) for all files.
|
|
4556
|
+
|
|
4557
|
+
Traverses the group hierarchy to find the top-most ancestor for each
|
|
4558
|
+
file and writes the corrected tindex value to wfile/sfile records.
|
|
4559
|
+
|
|
4560
|
+
Args:
|
|
4561
|
+
dsid -- dataset ID string
|
|
4562
|
+
act -- action bitmask controlling which file tables to update
|
|
4563
|
+
|
|
4564
|
+
Returns:
|
|
4565
|
+
Number of file records updated.
|
|
4566
|
+
"""
|
|
4092
4567
|
tcnt = 0
|
|
4093
4568
|
cgidxs = {}
|
|
4094
4569
|
if 'GI' in self.params:
|
|
@@ -4100,8 +4575,19 @@ class DsArch(PgArch, PgMeta):
|
|
|
4100
4575
|
tcnt += self.reset_top_gindex(dsid, 0, act)
|
|
4101
4576
|
return tcnt
|
|
4102
4577
|
|
|
4103
|
-
# set the re-archived file counts for groups
|
|
4104
4578
|
def set_rearchive_filenumber(self, dsid, bidx, total, act):
|
|
4579
|
+
"""
|
|
4580
|
+
Record progress of a re-archive operation in the dscheck tracking table.
|
|
4581
|
+
|
|
4582
|
+
Stores how many files have been processed so far (*bidx* out of *total*)
|
|
4583
|
+
so that interrupted re-archive jobs can resume from the right offset.
|
|
4584
|
+
|
|
4585
|
+
Args:
|
|
4586
|
+
dsid -- dataset ID string
|
|
4587
|
+
bidx -- index of the last successfully processed file
|
|
4588
|
+
total -- total file count for this archive operation
|
|
4589
|
+
act -- action type bitmask
|
|
4590
|
+
"""
|
|
4105
4591
|
if 'GI' in self.params:
|
|
4106
4592
|
lmt = bidx + 20
|
|
4107
4593
|
if lmt > total: lmt = total
|
|
@@ -4110,8 +4596,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
4110
4596
|
self.reset_group_filenumber(dsid, act)
|
|
4111
4597
|
self.CHGGRPS = {}
|
|
4112
4598
|
|
|
4113
|
-
# reset group metadata via scm
|
|
4114
4599
|
def reset_group_metadata(self, dsid, act):
|
|
4600
|
+
"""
|
|
4601
|
+
Regenerate dataset/group metadata (e.g. summary XML) via the scm tool.
|
|
4602
|
+
|
|
4603
|
+
Triggers metadata recomputation for all groups affected by recent file
|
|
4604
|
+
changes, using the act bitmask to determine which metadata types to update.
|
|
4605
|
+
|
|
4606
|
+
Args:
|
|
4607
|
+
dsid -- dataset ID string
|
|
4608
|
+
act -- action bitmask (e.g. 4 for web files, 8 for saved files)
|
|
4609
|
+
"""
|
|
4115
4610
|
dcnd = "dsid = '{}'".format(dsid)
|
|
4116
4611
|
gindices = sorted(self.CHGGRPS)
|
|
4117
4612
|
if gindices:
|
|
@@ -4130,12 +4625,28 @@ class DsArch(PgArch, PgMeta):
|
|
|
4130
4625
|
if act == 1 or act&4 and re.search(r'(Y|B|W)', pgrec['meta_link']):
|
|
4131
4626
|
self.pgsystem("{} -d {} -rw all".format(self.PGOPT['scm'], dsid, self.LOGWRN, 1029))
|
|
4132
4627
|
|
|
4133
|
-
# get web file name for given local file name
|
|
4134
4628
|
def get_archive_filename(self, lfile):
|
|
4629
|
+
"""
|
|
4630
|
+
Derive the archive (web/saved) file name from a local file path.
|
|
4631
|
+
|
|
4632
|
+
Strips any leading directory components and compression extensions so
|
|
4633
|
+
that only the base file name suitable for storage is returned.
|
|
4634
|
+
|
|
4635
|
+
Args:
|
|
4636
|
+
lfile -- local file path string
|
|
4637
|
+
|
|
4638
|
+
Returns:
|
|
4639
|
+
Archive file name string (basename, possibly without compression suffix).
|
|
4640
|
+
"""
|
|
4135
4641
|
return lfile if 'KP' in self.params else op.basename(lfile)
|
|
4136
4642
|
|
|
4137
|
-
# clean up local files and directories after action
|
|
4138
4643
|
def clean_local_files(self):
|
|
4644
|
+
"""
|
|
4645
|
+
Delete local staging files and empty directories after a successful action.
|
|
4646
|
+
|
|
4647
|
+
Removes all files listed in params['LF'] from the local filesystem and
|
|
4648
|
+
optionally prunes empty parent directories when -RD is set.
|
|
4649
|
+
"""
|
|
4139
4650
|
cnt = 0
|
|
4140
4651
|
if 'DD' in self.params: self.record_delete_directory(None, self.params['DD'])
|
|
4141
4652
|
for lfile in self.params['LF']:
|
|
@@ -4145,25 +4656,45 @@ class DsArch(PgArch, PgMeta):
|
|
|
4145
4656
|
self.pglog("cnt local files cleaned", self.PGOPT['emerol'])
|
|
4146
4657
|
if 'DD' in self.params: self.clean_delete_directory(self.PGOPT['wrnlog'])
|
|
4147
4658
|
|
|
4148
|
-
# transfer cached self.ERRMSG between globally and locally
|
|
4149
4659
|
def reset_errmsg(self, errcnt):
|
|
4660
|
+
"""
|
|
4661
|
+
Flush the accumulated error message buffer and update ERRCNT.
|
|
4662
|
+
|
|
4663
|
+
When *errcnt* >= 0, appends self.ERRMSG to the log and clears it.
|
|
4664
|
+
When *errcnt* == -1, logs any remaining message without updating the count.
|
|
4665
|
+
|
|
4666
|
+
Args:
|
|
4667
|
+
errcnt -- current error count to accumulate into self.ERRCNT, or -1 to flush only
|
|
4668
|
+
|
|
4669
|
+
Returns:
|
|
4670
|
+
Updated cumulative error count (self.ERRCNT).
|
|
4671
|
+
"""
|
|
4150
4672
|
ret = 0
|
|
4151
4673
|
if errcnt < 0: # cache self.ERRMSG globally
|
|
4152
|
-
self.PGLOG['
|
|
4153
|
-
self.PGLOG['
|
|
4674
|
+
self.PGLOG['ERRCNT'] += self.ERRCNT
|
|
4675
|
+
self.PGLOG['ERRMSG'] += self.ERRMSG
|
|
4154
4676
|
self.ERRCNT = 0
|
|
4155
4677
|
self.ERRMSG = ''
|
|
4156
4678
|
else:
|
|
4157
4679
|
if errcnt > 0: # cache self.ERRMSG locally
|
|
4158
|
-
self.ERRMSG += self.PGLOG['
|
|
4680
|
+
self.ERRMSG += self.PGLOG['ERRMSG']
|
|
4159
4681
|
self.ERRCNT += errcnt
|
|
4160
4682
|
ret = 1
|
|
4161
|
-
self.PGLOG['
|
|
4162
|
-
self.PGLOG['
|
|
4683
|
+
self.PGLOG['ERRCNT'] = 0
|
|
4684
|
+
self.PGLOG['ERRMSG'] = ''
|
|
4163
4685
|
return ret
|
|
4164
4686
|
|
|
4165
|
-
# copy a file to a alternate destination
|
|
4166
4687
|
def copy_alter_local(self, wfile, ahome):
|
|
4688
|
+
"""
|
|
4689
|
+
Copy a web file to an alternate GLADE location.
|
|
4690
|
+
|
|
4691
|
+
Used to mirror files to a secondary directory (e.g. CGD or alternate
|
|
4692
|
+
web home) when the dataset is configured to maintain duplicates.
|
|
4693
|
+
|
|
4694
|
+
Args:
|
|
4695
|
+
wfile -- relative web file path
|
|
4696
|
+
ahome -- alternate root directory to copy into
|
|
4697
|
+
"""
|
|
4167
4698
|
afile = wfile
|
|
4168
4699
|
bproc = self.PGSIG['BPROC']
|
|
4169
4700
|
afile = re.sub(self.PGLOG['DSDHOME'], ahome, afile)
|
|
@@ -4171,8 +4702,17 @@ class DsArch(PgArch, PgMeta):
|
|
|
4171
4702
|
self.local_copy_local(afile, wfile, self.PGOPT['emerol']|self.OVERRIDE)
|
|
4172
4703
|
if bproc != self.PGSIG['BPROC']: self.PGSIG['BPROC'] = bproc
|
|
4173
4704
|
|
|
4174
|
-
# delete a file at alternate location
|
|
4175
4705
|
def delete_alter_local(self, wfile, ahome):
|
|
4706
|
+
"""
|
|
4707
|
+
Delete a web file from an alternate GLADE location.
|
|
4708
|
+
|
|
4709
|
+
Removes the mirrored copy created by copy_alter_local() when a file
|
|
4710
|
+
is deleted or moved out of the alternate web home.
|
|
4711
|
+
|
|
4712
|
+
Args:
|
|
4713
|
+
wfile -- relative web file path
|
|
4714
|
+
ahome -- alternate root directory where the copy resides
|
|
4715
|
+
"""
|
|
4176
4716
|
afile = wfile
|
|
4177
4717
|
bproc = self.PGSIG['BPROC']
|
|
4178
4718
|
afile = re.sub(self.PGLOG['DSDHOME'], ahome, afile)
|
|
@@ -4180,8 +4720,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
4180
4720
|
if op.exists(afile): self.delete_local_file(afile, self.PGOPT['emerol'])
|
|
4181
4721
|
if bproc != self.PGSIG['BPROC']: self.PGSIG['BPROC'] = bproc
|
|
4182
4722
|
|
|
4183
|
-
# get version information
|
|
4184
4723
|
def get_version_info(self):
|
|
4724
|
+
"""
|
|
4725
|
+
Retrieve and display version control records from RDADB.
|
|
4726
|
+
|
|
4727
|
+
Queries the dsvrsn table for the current dataset and writes formatted
|
|
4728
|
+
version information to self.OUTPUT.
|
|
4729
|
+
"""
|
|
4185
4730
|
tname = "dsvrsn"
|
|
4186
4731
|
dsid = self.params['DS']
|
|
4187
4732
|
hash = self.TBLHASH[tname]
|
|
@@ -4203,8 +4748,20 @@ class DsArch(PgArch, PgMeta):
|
|
|
4203
4748
|
else:
|
|
4204
4749
|
self.pglog("No version control information retrieved", self.PGOPT['wrnlog'])
|
|
4205
4750
|
|
|
4206
|
-
# get a DOI number for given dsid
|
|
4207
4751
|
def get_new_version(self, nrec, doi):
|
|
4752
|
+
"""
|
|
4753
|
+
Fetch or generate a DOI number for a new version record.
|
|
4754
|
+
|
|
4755
|
+
If *doi* is not yet assigned, queries the DOI service and stores the
|
|
4756
|
+
result in *nrec*.
|
|
4757
|
+
|
|
4758
|
+
Args:
|
|
4759
|
+
nrec -- mutable new version record dict to update with the DOI
|
|
4760
|
+
doi -- existing DOI string, or None to request a new one
|
|
4761
|
+
|
|
4762
|
+
Returns:
|
|
4763
|
+
DOI string (may be the same as *doi* if already set).
|
|
4764
|
+
"""
|
|
4208
4765
|
nrec['doi'] = doi
|
|
4209
4766
|
if not ('start_date' in nrec and nrec['start_date']): nrec['start_date'] = self.curdate()
|
|
4210
4767
|
if not ('start_time' in nrec and nrec['start_time']): nrec['start_time'] = self.curtime()
|
|
@@ -4213,8 +4770,18 @@ class DsArch(PgArch, PgMeta):
|
|
|
4213
4770
|
nrec['iversion'] = 1
|
|
4214
4771
|
return nrec
|
|
4215
4772
|
|
|
4216
|
-
# replace old version record with a new one
|
|
4217
4773
|
def transfer_version_info(self, nrec, orec, doi):
|
|
4774
|
+
"""
|
|
4775
|
+
Replace the file associations of an old version with those of a new version.
|
|
4776
|
+
|
|
4777
|
+
Migrates all wfile/sfile records linked to *orec* over to *nrec*,
|
|
4778
|
+
updates status flags on the old version, and assigns the DOI.
|
|
4779
|
+
|
|
4780
|
+
Args:
|
|
4781
|
+
nrec -- new version record dict (destination)
|
|
4782
|
+
orec -- old version record dict (source)
|
|
4783
|
+
doi -- DOI string to assign to the new version
|
|
4784
|
+
"""
|
|
4218
4785
|
dsid = self.params['DS']
|
|
4219
4786
|
vinfo = "Version Control {}".format(orec['vindex'])
|
|
4220
4787
|
if not ('start_date' in nrec and nrec['start_date']): nrec['start_date'] = self.curdate()
|
|
@@ -4237,8 +4804,15 @@ class DsArch(PgArch, PgMeta):
|
|
|
4237
4804
|
self.pglog("{} {}: Set status to 'H'".format(dsid, vinfo), self.PGOPT['wrnlog'])
|
|
4238
4805
|
return nrec
|
|
4239
4806
|
|
|
4240
|
-
# add or modify version control information
|
|
4241
4807
|
def set_version_info(self):
|
|
4808
|
+
"""
|
|
4809
|
+
Insert or update version control records in RDADB.
|
|
4810
|
+
|
|
4811
|
+
Handles both new version creation (-NV) and modification of existing
|
|
4812
|
+
version records. When creating a new version from an existing one,
|
|
4813
|
+
calls transfer_version_info() to migrate file associations.
|
|
4814
|
+
Resets the RDADB version counter after changes.
|
|
4815
|
+
"""
|
|
4242
4816
|
tname = "dsvrsn"
|
|
4243
4817
|
dsid = self.params['DS']
|
|
4244
4818
|
dcnd = "dsid = '{}'".format(dsid)
|
|
@@ -4329,8 +4903,13 @@ class DsArch(PgArch, PgMeta):
|
|
|
4329
4903
|
self.pglog("{}: Relinked {} Saved file record{}".format(vinfo, fcnt, s), self.PGOPT['wrnlog'])
|
|
4330
4904
|
self.pglog("{}/{} of {} added/modified in RDADB!".format(addcnt, modcnt, msg), self.PGOPT['wrnlog'])
|
|
4331
4905
|
|
|
4332
|
-
# terminate version control information for given version indices
|
|
4333
4906
|
def terminate_version_info(self):
|
|
4907
|
+
"""
|
|
4908
|
+
Mark version control records as terminated (-TV action).
|
|
4909
|
+
|
|
4910
|
+
Sets the status of each version index in params['VI'] to 'A' (archived)
|
|
4911
|
+
and records a termination date, then resets the RDADB version counter.
|
|
4912
|
+
"""
|
|
4334
4913
|
msg = "{} Version Control".format(self.ALLCNT)
|
|
4335
4914
|
if self.ALLCNT > 1: msg += "s"
|
|
4336
4915
|
self.pglog("Terminate {} ...".format(msg), self.WARNLG)
|
|
@@ -4372,8 +4951,12 @@ class DsArch(PgArch, PgMeta):
|
|
|
4372
4951
|
self.pglog("{}/{} of {} terminated for DOI/Version Control".format(doicnt, modcnt, msg), self.PGOPT['wrnlog'])
|
|
4373
4952
|
if delcnt: self.pglog("{}/{} of {} deleted".format(delcnt, modcnt, msg), self.PGOPT['wrnlog'])
|
|
4374
4953
|
|
|
4375
|
-
# main function to excecute this script
|
|
4376
4954
|
def main():
|
|
4955
|
+
"""Entry point for the dsarch command-line utility.
|
|
4956
|
+
|
|
4957
|
+
Instantiates DsArch, parses arguments via read_parameters(), executes the
|
|
4958
|
+
requested action via start_actions(), and exits cleanly.
|
|
4959
|
+
"""
|
|
4377
4960
|
object = DsArch()
|
|
4378
4961
|
object.read_parameters()
|
|
4379
4962
|
object.start_actions()
|