rda-python-dsarch 1.0.19__tar.gz → 1.0.21__tar.gz

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 1.0.19
+Version: 1.0.21
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsarch"
-version = "1.0.19"
+version = "1.0.21"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/src/rda_python_dsarch/PgArch.py

@@ -151,6 +151,7 @@ PgOPT.OPTS = {                         # (!= 0) - setting actions
    'ET' : [2, 'EndTime',      32],
    'EV' : [2, 'ExternalVersion', 1],
    'FD' : [2, 'FileDate',    257],
+   'FF' : [2, 'FileFlag',      1, 'FP'],
    'FS' : [2, 'FileStatus',    1, 'PI'],
    'FT' : [2, 'FileTime',     33],
    'GI' : [2, 'GroupIndex',   17],
@@ -196,7 +197,7 @@ PgOPT.OPTS = {                         # (!= 0) - setting actions
 }
 
 PgOPT.ALIAS = {
-   'AF' : ['FF', 'FileFormat', 'ExternalFormat'],
+   'AF' : ['FileFormat', 'ExternalFormat'],
    'AL' : ['AsynchronousLimit'],
    'AO' : ['ActOption'],
    'AH' : ['ArchiveHelp', 'ArchHelp', 'ArchHelpFile'],
@@ -332,6 +333,7 @@ PgOPT.TBLHASH['wfile'] = {
    'Q' : ['QT', "btype",               1, "bfile.type"],
    'L' : ['LC', "locflag",             1],
    'H' : ['FS', "wfile.status",        1],
+   'P' : ['FF', "fileflag",            1],
    'S' : ['SZ', "wfile.data_size",     0],
    'J' : ['FD', "wfile.date_modified", 1],
    'K' : ['FT', "wfile.time_modified", 1],
@@ -354,6 +356,7 @@ PgOPT.TBLHASH['sfile'] = {
    'Q' : ['QT', "btype",               1, "bfile.type"],
    'L' : ['LC', "locflag",             1],
    'H' : ['FS', "sfile.status",        1],
+   'P' : ['FF', "fileflag",            1],
    'S' : ['SZ', "sfile.data_size",     0],
    'J' : ['FD', "sfile.date_modified", 1],
    'K' : ['FT', "sfile.time_modified", 1],
@@ -383,6 +386,7 @@ PgOPT.TBLHASH['hfile'] = {
    'N' : ['DF', "data_format",     1],
    'O' : ['DO', "disp_order",     -1],
    'H' : ['FS', "status",          1],
+   'P' : ['FF', "fileflag",        1],
    'S' : ['SZ', "data_size",       0],
    'R' : ['SR', "source",          1],
    'I' : ['ID', "init_date",       1],
@@ -411,10 +415,10 @@ PgOPT.PGOPT['MSET']  = "SA"
 PgOPT.PGOPT['dataset'] = "SWFULQA"
 PgOPT.PGOPT['dsperiod'] = "GJKXY"
 PgOPT.PGOPT['dsgroup'] = "IGXTPQASW"
-PgOPT.PGOPT['wfile'] = "FTIVMNLHS"
-PgOPT.PGOPT['sfile'] = "FTIVMNLHS"
+PgOPT.PGOPT['wfile'] = "FTIVMNLHPS"
+PgOPT.PGOPT['sfile'] = "FTIVMNLHPS"
 PgOPT.PGOPT['bfile'] = "FNMTHS"
-PgOPT.PGOPT['hfile'] = 'FMNLTHSU'
+PgOPT.PGOPT['hfile'] = 'FMNLTHPSU'
 PgOPT.PGOPT['dsvrsn'] = "VIEDSJX"
 PgOPT.PGOPT['UACTS'] = (PgOPT.OPTS['AQ'][0]|PgOPT.OPTS['SD'][0]|PgOPT.OPTS['DL'][0])
 
@@ -422,10 +426,10 @@ PgOPT.PGOPT['UACTS'] = (PgOPT.OPTS['AQ'][0]|PgOPT.OPTS['SD'][0]|PgOPT.OPTS['DL']
 PgOPT.PGOPT['dsall'] = "TSHWFULQAVCBMPIDNGJKXYIJKXY"   # include 'pdall'
 PgOPT.PGOPT['pdall'] = "GJKXY"
 PgOPT.PGOPT['gpall'] = "IGXTRQAPSWMDN"
-PgOPT.PGOPT['wfall'] = "FTCIXVMNOBQLHSJKAED"
-PgOPT.PGOPT['sfall'] = "FTCIXVMNOBQLHSJKD"
+PgOPT.PGOPT['wfall'] = "FTCIXVMNOBQLHPSJKAED"
+PgOPT.PGOPT['sfall'] = "FTCIXVMNOBQLHPSJKD"
 PgOPT.PGOPT['bfall'] = "FNMOTHSCJKD"
-PgOPT.PGOPT['hfall'] = 'FTLCMNOHSRIJKUD'
+PgOPT.PGOPT['hfall'] = 'FTLCMNOHPSRIJKUD'
 PgOPT.PGOPT['vrsnall'] = "VIEDSJKXYN"
 
 # PgOPT.PGOPT['ofile'] reserved for original type 'B' hpps file name
@@ -693,6 +697,7 @@ def check_enough_options(cact, acts):
       if 'EM' not in PgOPT.params: PgLOG.PGLOG['LOGMASK'] &= ~PgLOG.EMLALL   # turn off email if not yet
       PgOPT.PGOPT['emerol'] |= PgLOG.EXITLG
       # set command line Batch options
+      set_qsub_walltime()
       PgCMD.set_batch_options(PgOPT.params, 1)
       PgCMD.init_dscheck(0, '', "dsarch", PgOPT.params['DS'], cact, PgLOG.PGLOG['CURDIR'], PgOPT.params['LN'],
                          PgOPT.params['BP'], PgOPT.PGOPT['emerol'])
@@ -700,6 +705,30 @@ def check_enough_options(cact, acts):
 
    PgSIG.start_none_daemon('dsarch', PgOPT.PGOPT['CACT'], PgOPT.params['LN'], 1, PgOPT.params['WI'], 1, PgOPT.params['AL'])
 
+#
+# set qsub walltime if needed
+#
+def set_qsub_walltime():
+
+   fcnt = 0
+   if 'LF' in PgOPT.params:
+      fcnt = len(PgOPT.params['LF'])
+   elif 'WF' in PgOPT.params:
+      fcnt = len(PgOPT.params['WF'])
+   else:
+      return
+
+   if 'GX' in PgOPT.params: fcnt *= 30
+   hr = int(fcnt/30000)
+   if hr == 6: return
+
+   if hr < 2:
+      hr = 2
+   elif hr > 24:
+      hr = 24
+   PgCMD.set_one_boption('qoptions', '-l walltime={}:00:00'.format(hr), 1)
+   return
+
 #
 # build local file list for given local directory and reset file sizes
 #

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/src/rda_python_dsarch/dsarch.py

@@ -518,7 +518,7 @@ def archive_web_files():
             elif pgrec:
                info = get_file_origin_info(wfile, pgrec)
             elif locflag == 'O':
-               info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
+               info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])               
             wid = set_one_webfile(i, pgrec, wfile, fnames, type, info)
             if not wid:
                PgOPT.params['LF'][i] = PgOPT.params['WF'][i] = None
@@ -1673,7 +1673,7 @@ def tar_backup_savedfiles(tarfile, tinfo, ccnt, chkstat):
       else:
          tardir = "{}/{}".format(dshome, stype)
          savedfile = "{}/{}".format(tardir, sfile) if relative else sfile
-      if not op.isfile(savedfile):
+      if not op.exists(savedfile):
          PgLOG.pglog(savedfile + ": Saved file not exists to backup", PgOPT.PGOPT['extlog'])
       if relative:
          tarcmds[i] = "tar -{}vf {} -C {} {}".format(topt, tarfile, tardir, sfile)
@@ -1761,7 +1761,7 @@ def tar_backup_webfiles(tarfile, tinfo, ccnt, chkstat):
       else:
          tardir = dshome
          webfile = "{}/{}".format(tardir, wfile) if relative else wfile
-      if not op.isfile(webfile):
+      if not op.exists(webfile):
          PgLOG.pglog(webfile + ": Web file not exists to backup", PgOPT.PGOPT['extlog'])
       if relative:
          tarcmds[i] = "tar -{}vf {} -C {} {}".format(topt, tarfile, tardir, wfile)
@@ -2869,6 +2869,8 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
             record['status'] = 'I'
       if 'checksum' in record: PgLOG.pglog("md5({}-{})={}".format(dsid, file, record['checksum']), PgLOG.LOGWRN)
       if 'gindex' in record and record['gindex']: record['tindex'] = PgMeta.get_top_gindex(dsid, record['gindex'])
+      stat = check_file_flag(file, info, record, pgrec)
+      if not stat: return stat
       if pgrec:
          if pgrec['wfile'] != file: record['wfile'] = file
          if 'bid' in record and record['bid'] and pgrec['bid'] and record['bid'] != pgrec['bid']:
@@ -2890,8 +2892,9 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
          if 'status' not in record: record['status'] = 'P'
          if not info:
             info = PgFile.check_local_file(PgArch.get_web_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
+            stat = check_file_flag(file, info, record)
+            if not stat: return stat
          if info:
-            if not info['isfile']: return PgLOG.pglog("file: is a directory", PgOPT.PGOPT['emlerr'])
             record['data_size'] = info['data_size']
             record['date_modified'] = info['date_modified']
             record['time_modified'] = info['time_modified']
@@ -2912,6 +2915,22 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
 
    return wid
 
+#
+# check 
+#
+def check_file_flag(file, info, record, pgrec = None):
+
+   if not info: return PgLOG.SUCCESS
+
+   fflag = 'F' if info['fileflag'] else 'P'
+   if 'fileflag' in record:
+      if fflag != record['fileflag']:
+         return PgLOG.pglog("{}: Cannot set File Flag '{}' to '{}'".format(file, fflag, record['fileflag']), PgOPT.PGOPT['emlerr'])
+   elif not (pgrec and pgrec['fileflag'] == fflag):
+      record['fileflag'] = fflag
+
+   return PgLOG.SUCCESS
+
 #
 # add or modify Help file information
 #
@@ -3010,6 +3029,8 @@ def set_one_helpfile(i, pgrec, file, flds, type, info = None, ndsid = None):
       if 'url' in record:
          record['locflag'] = 'R'
          record['date_modified'] = PgUtil.curdate()
+      stat = check_file_flag(file, info, record, pgrec)
+      if not stat: return stat
       if pgrec:
          if pgrec['hfile'] != file: record['hfile'] = file
          if not ndsid:
@@ -3030,8 +3051,9 @@ def set_one_helpfile(i, pgrec, file, flds, type, info = None, ndsid = None):
          if 'status' not in record: record['status'] = 'P'
          if not (info or 'url' in record):
             info = PgFile.check_local_file(PgArch.get_help_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
+            stat = check_file_flag(file, info, record, pgrec)
+            if not stat: return stat
          if info:
-            if not info['isfile']: return PgLOG.pglog("file: is a directory", PgOPT.PGOPT['emlerr'])
             record['data_size'] = info['data_size']
             record['date_modified'] = info['date_modified']
             record['time_modified'] = info['time_modified']
@@ -3188,6 +3210,8 @@ def set_one_savedfile(i, pgrec, file, flds, type, info = None, ndsid = None, sac
             record['status'] = 'I'
       if 'checksum' in record and record['checksum']: PgLOG.pglog("md5({}-{})={}".format(dsid, file, record['checksum']), PgLOG.LOGWRN)
       if 'gindex' in record and record['gindex']: record['tindex'] = PgMeta.get_top_gindex(dsid, record['gindex'])
+      stat = check_file_flag(file, info, record, pgrec)
+      if not stat: return stat
       if pgrec:
          if pgrec['sfile'] != file: record['sfile'] = file
          if not ndsid:
@@ -3213,8 +3237,9 @@ def set_one_savedfile(i, pgrec, file, flds, type, info = None, ndsid = None, sac
          if 'status' not in record: record['status'] = 'P'
          if not info:
             info = PgFile.check_local_file(PgArch.get_saved_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
+            stat = check_file_flag(file, info, record, pgrec)
+            if not stat: return stat
          if info:
-            if not info['isfile']: return PgLOG.pglog(file + ": is a directory", PgOPT.PGOPT['emlerr'])
             record['data_size'] = info['data_size']
             record['date_modified'] = info['date_modified']
             record['time_modified'] = info['time_modified']
@@ -4445,7 +4470,7 @@ def web_to_saved_files():
 #
 def get_file_origin_info(fname, pgrec):
 
-   info = {'isfile' : 1, 'data_size' : pgrec['data_size']}
+   info = {'isfile' : (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size' : pgrec['data_size']}
    info['fname'] = op.basename(fname)
    info['date_modified'] = pgrec['date_modified']
    info['time_modified'] = pgrec['time_modified']

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/src/rda_python_dsarch/dsarch.usg

@@ -734,6 +734,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
          [-(PO|PatternOffset) PatternStringOffset]
          [-(ST|SavedFileType) SavedFileTypes]   # Primary, VersionControl, Work, oRiginal
          [-(FS|FileStatus) SavedFileStatus]     # Public or Interal file
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]  # CGD data; Glade, Object or Both
          [-(DF|DataFormat) DataFormats]         # BINARY, ASCII, NetCDF and etc.
          [-(AF|ArchiveFormat) FileArchiveFormats]   # tarred, Compressed and etc.
@@ -796,6 +797,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
          [-(SF|SavedFile) SavedFileNames]
          [-(ST|SavedFileType) SavedFileTypes]
          [-(FS|FileStatus) SavedFileStatus]
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]
          [-(GI|GroupIndex) GroupIndices]
          [-(GN|GroupName) GroupNames]
@@ -831,6 +833,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
   Q      -(QT|QuasarType)      Quasar backyp file types
   L      -(LC|Location)        Saved file storage location flag
   H      -(FS|FileStatus)      Saved file archiving status
+  P      -(FF|FileFlag)        File type flag, File or Path
   S      -(SZ|Size)            Saved file sizes
   J      -(FD|FileDate)        date data last modified on
   K      -(FT|FileTime)        time data last modified at 
@@ -866,6 +869,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
          [-(WT|WebFileType) WebFileTypes]     # Web Data or NCAR Data
          [-(LC|Location) StorageLocationFlags]
          [-(FS|FileStatus) WebFileStatus]     # Public or interal file
+         [-(FF|FileFlag) FilesOrPaths]        # file type flag, File or Path
          [-(DF|DataFormat) DataFormats]       # BINARY, ASCII, NetCDF and etc.
          [-(AF|ArchiveFormat) FileArchiveFormats]  # tarred, Compressed and etc.
          [-(DO|DisplayOrder] DisplayOrderIndices]
@@ -940,6 +944,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
          [-(WF|WebFile) WebFileNames]
          [-(WT|WebFileType) WebFileTypes]
          [-(FS|FileStatus) WebFileStatus]
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]
          [-(GI|GroupIndex) GroupIndices]
          [-(GN|GroupName) GroupNames]
@@ -975,6 +980,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
   Q      -(QT|QuasarType)      Quasar backyp fie types
   L      -(LC|Location)        Web file storage location flag
   H      -(FS|FileStatus)      Web file archiving status
+  P      -(FF|FileFlag)        File type flag, File or Path
   S      -(SZ|Size)            Web file sizes
   J      -(FD|FileDate)        date data last modified on
   K      -(FT|FileTime)        time data last modified at 
@@ -1021,10 +1027,11 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
          [-(SR|Source) ContributionSource]
          [-(ON|OrderNames) OrderNameString]
          [-(OB|OrderBy) OrderByPattern]
-         [-(HT|HelpFileType) HelpFileTypes]  # Document or Software
-         [-(FS|FileStatus) HelpFileStatus]     # Public or Interal file
+         [-(HT|HelpFileType) HelpFileTypes]     # Document or Software
+         [-(FS|FileStatus) HelpFileStatus]      # Public or Interal file
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]  # Glade, Object, Both or Remote
-         [-(DF|DataFormat) DataFormats]       # Python, PDF, and etc.
+         [-(DF|DataFormat) DataFormats]         # Python, PDF, and etc.
          [-(AF|ArchiveFormat) FileArchiveFormats]  # tarred, Compressed and etc.
          [-(DO|DisplayOrder] DisplayOrderIndices]
          [-(SZ|Size) FileSizes]
@@ -1075,6 +1082,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
          [-(HF|HelpFile) HelpFileNames]
          [-(HT|HelpFileType) HelpFileTypes]     # Document or Software
          [-(FS|FileStatus) SavedFileStatus]
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]
          [-(SR|Source) ContributionSource]
          [-(DE|Description) SavedFileDescriptions]
@@ -1098,6 +1106,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
   O      -(DO|DisplayOrder)    display order indices
   T      -(HT|HelpFileType)    Help file archiving types
   H      -(FS|FileStatus)      Help file archiving status
+  P      -(FF|FileFlag)        File type flag, File or Path
   L      -(LC|Location)        Help file storage location flag
   S      -(SZ|Size)            Help file sizes
   R      -(SR|Source)          Help file contribution source
@@ -1235,6 +1244,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
          [-(PO|PatternOffset) PatternStringOffset]
          [-(ST|SavedFileType) SavedFileTypes]
          [-(FS|FileStatus) SavedFileStatus]       # P-Public or I-Internal
+         [-(FF|FileFlag) FilesOrPaths]            # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]
          [-(DF|DataFormat) DataFormat]            # BINARY, ASCII, NetCDF and etc.
          [-(AF|ArchiveFormat) FileArchiveFormat]  # tarred, compressed and etc. 
@@ -1334,16 +1344,17 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
   dsarch [-(DS|dataset)] dsnnn.n -(AW|ArchiveWebFile) [Mode Options]
          [-(LF|LocalFile) LocalFileNames]  # mandatory unless -LD or -LL present
          [-(LD|LocalDirectory) LocalDirectoryName]  # directory for local file names
-         [-(LL|LocalFileList) LocalFileListName]  # contains a list of local file names
-         [-(WF|WebFile) webFileNames]      # default to -LF
+         [-(LL|LocalFileList) LocalFileListName]    # contains a list of local file names
+         [-(WF|WebFile) webFileNames]               # default to -LF
          [-(GI|GroupIndex) GroupIndices]
          [-(TG|TopGroupIndex) TopGroupIndices]
          [-(GN|GroupName) GroupNames]
          [-(PO|PatternOffset) PatternStringOffset]
-         [-(WT|WebFileType) WebFileTypes]  # Data, Ncar Data
-         [-(FS|FileStatus) WebFileStatus]  # P-Public or I-Internal
+         [-(WT|WebFileType) WebFileTypes]       # Data, Ncar Data
+         [-(FS|FileStatus) WebFileStatus]       # P-Public or I-Internal
+         [-(FF|FileFlag) FilesOrPaths]          # file type flag, File or Path
          [-(LC|Location) StorageLocationFlags]
-         [-(DF|DataFormat) DataFormat]     # BINARY, ASCII, NetCDF and etc.
+         [-(DF|DataFormat) DataFormat]            # BINARY, ASCII, NetCDF and etc.
          [-(AF|ArchiveFormat) FileArchiveFormat]  # tarred, compressed and etc. 
          [-(DO|DisplayOrder] DisplayOrderIndices]
          [-(SZ|Size) FileSizes]
@@ -1449,6 +1460,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
          [-(HF|HelpFile) HelpFileNames]            # default to -LF
          [-(HT|HelpFileType) HelpFileTypes]
          [-(FS|FileStatus) HelpFileStatus]         # P-Public or I-Internal
+         [-(FF|FileFlag) FilesOrPaths]             # file type flag, File or Path
          [-(DF|DataFormat) FileContentFormat]      # PDF, Python and etc.
          [-(AF|ArchiveFormat) FileArchiveFormat]   # tarred, compressed and etc. 
          [-(DO|DisplayOrder] DisplayOrderIndices]
@@ -2222,6 +2234,8 @@ Information options are used to pass information, one or multiple values, into
 
   -FD or -FileDate, for date of a data file last modified on. 
 
+  -FF or -FileFlag, File type flag, F for File and P for Path.
+
   -FS or -FileStatus, File status, a single letter status flag, P - show file
   to public users; and I - to show file to specialist only.
  

{rda_python_dsarch-1.0.19 → rda_python_dsarch-1.0.21}/src/rda_python_dsarch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 1.0.19
+Version: 1.0.21
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch