rda-python-dsarch 1.0.14__tar.gz → 1.0.16__tar.gz

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 1.0.14
+Version: 1.0.16
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "rda_python_dsarch"
-version = "1.0.14"
+version = "1.0.16"
 authors = [
   { name="Zaihua Ji", email="zji@ucar.edu" },
 ]

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/src/rda_python_dsarch/PgArch.py

@@ -141,6 +141,7 @@ PgOPT.OPTS = {                         # (!= 0) - setting actions
    'BP' : [2, 'BatchProcess',  0, ''],
    'BS' : [2, 'BackStatus',    1, 'NTA'],
    'BT' : [2, 'BeginTime',    32],
+   'DA' : [2, 'DataAccess',    1, 'BCFN'],
    'DB' : [2, 'Debug',         0],
    'DE' : [2, 'Description',  65],
    'DF' : [2, 'DataFormat',    1],
@@ -205,6 +206,7 @@ PgOPT.ALIAS = {
    'BG' : ['b'],
    'BP' : ['d', 'DelayedMode'],
    'CL' : ['CleanLocFile', 'CleanLocalFile'],
+   'DA' : ['DataAccessflag', 'FileListFlag'],
    'DD' : ['DeleteDirLevel', 'DeleteEmptyDir'],
    'DE' : ['Desc', 'Note', 'FileDesc', 'FileDescription'],
    'DF' : ['TF', 'ContentFormat'],
@@ -281,10 +283,11 @@ PgOPT.TBLHASH['dataset'] = {         # condition flag, 0-int, 1-string, -1-exclu
    'U' : ['UD', "use_rdadb",    1],
    'L' : ['LC', "locflag",      1],
    'Q' : ['BF', "backflag",     1],
+   'A' : ['DA', "accessflag",   1],
    'V' : ['GL', "grouplevel",   0],
    'C' : ['FL', "filelimit",    0],
    'B' : ['BL', "buttonlimit",  0],
-   'A' : ['ML', "meta_link",   -1],
+   'M' : ['ML', "meta_link",   -1],
    'P' : ['NW', "wnote",       -1],
    'I' : ['NI', "inote",       -1],
    'D' : ['ND', "dnote",       -1],
@@ -306,10 +309,11 @@ PgOPT.TBLHASH['dsgroup'] = {
    'T' : ['TI', "title",     1],
    'R' : ['GT', "grptype",   1],
    'Q' : ['BF', "backflag",  1],
+   'A' : ['DA', "accessflag",   1],
    'P' : ['GP', "pattern",   1],
    'S' : ['SP', "savedpath", 1],
    'W' : ['WP', "webpath",   1],
-   'A' : ['ML', "meta_link", 1],
+   'M' : ['ML', "meta_link", 1],
    'D' : ['NW', "wnote",     1],
    'N' : ['NI', "inote",     1]
  }
@@ -404,9 +408,9 @@ PgOPT.TBLHASH['dsvrsn'] = {
 # global info to be used by the whole application
 PgOPT.PGOPT['MSET']  = "SA"
 #default fields for getting info
-PgOPT.PGOPT['dataset'] = "SWFULQ"
+PgOPT.PGOPT['dataset'] = "SWFULQA"
 PgOPT.PGOPT['dsperiod'] = "GJKXY"
-PgOPT.PGOPT['dsgroup'] = "IGXTPQSW"
+PgOPT.PGOPT['dsgroup'] = "IGXTPQASW"
 PgOPT.PGOPT['wfile'] = "FTIVMNLHS"
 PgOPT.PGOPT['sfile'] = "FTIVMNLHS"
 PgOPT.PGOPT['bfile'] = "FNMTHS"
@@ -415,9 +419,9 @@ PgOPT.PGOPT['dsvrsn'] = "VIEDSJX"
 PgOPT.PGOPT['UACTS'] = (PgOPT.OPTS['AQ'][0]|PgOPT.OPTS['SD'][0]|PgOPT.OPTS['DL'][0])
 
 #all fields for getting info
-PgOPT.PGOPT['dsall'] = "TSHWFULQVCBAPIDNGJKXYIJKXY"   # include 'pdall'
+PgOPT.PGOPT['dsall'] = "TSHWFULQAVCBMPIDNGJKXYIJKXY"   # include 'pdall'
 PgOPT.PGOPT['pdall'] = "GJKXY"
-PgOPT.PGOPT['gpall'] = "IGXTRQPSWADN"
+PgOPT.PGOPT['gpall'] = "IGXTRQAPSWMDN"
 PgOPT.PGOPT['wfall'] = "FTCIXVMNOBQLHSJKAED"
 PgOPT.PGOPT['sfall'] = "FTCIXVMNOBQLHSJKD"
 PgOPT.PGOPT['bfall'] = "FNMOTHSCJKD"

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/src/rda_python_dsarch/PgMeta.py

@@ -592,7 +592,7 @@ def process_meta_gather(cate, logact = PgLOG.LOGWRN):
          cmd += ary[0]
          if PgSIG.start_background(cmd, act, opt, 1):
             cnt += 1
-            if PgLOG.PGLOG['DSCHECK'] and cnt > 0 and (cnt%20) == 0:
+            if PgLOG.PGLOG['DSCHECK'] and cnt > 0 and (cnt%10) == 0:
                PgCMD.add_dscheck_dcount(10, 0, logact)
          elif PgLOG.PGLOG['SYSERR']:
             PgDBI.record_dscheck_error(PgLOG.PGLOG['SYSERR'])
@@ -608,9 +608,6 @@ def process_meta_gather(cate, logact = PgLOG.LOGWRN):
             for tidx in TIDXS:
                PgLOG.pgsystem("{}{}".format(sx, tidx), act, opt)
 
-      if PgLOG.PGLOG['DSCHECK'] and (cnt%20) > 0:
-         PgCMD.add_dscheck_dcount(int((cnt%20)/2), 0, logact)
-
    del META[c]
    TIDXS = {}
    switch_logfile()

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/src/rda_python_dsarch/dsarch.py

@@ -390,11 +390,11 @@ def archive_web_files():
    while True:
       for i in range(bidx, ALLCNT):
          if PgSIG.PGSIG['BPROC'] < 2 and i > bidx and ((i-bidx)%20) == 0:
-            if PgLOG.PGLOG['DSCHECK']:
-               PgCMD.set_dscheck_dcount(i, chksize, PgOPT.PGOPT['extlog'])
             if metacnt >= PgOPT.PGOPT['RSMAX']:
                metatotal += PgMeta.process_metadata("W", metacnt, PgOPT.PGOPT['emerol'])
                metacnt = 0
+            if PgLOG.PGLOG['DSCHECK'] and metacnt == 0:
+               PgCMD.set_dscheck_dcount(i, chksize, PgOPT.PGOPT['extlog'])
             if 'EM' in PgOPT.params:
                PgLOG.PGLOG['PRGMSG'] = "{}/{} of {} web file{} archived/processed".format(acnt, i, ALLCNT, s)
          lfile = locfile = PgOPT.params['LF'][i]
@@ -2760,11 +2760,11 @@ def set_webfile_info(include = None):
    reorder = metatotal = metacnt = ADDCNT = MODCNT = 0
    for i in range(bidx, ALLCNT):
       if i > bidx and ((i-bidx)%20) == 0:
-         if PgLOG.PGLOG['DSCHECK']:
-            PgCMD.add_dscheck_dcount(20, 0, PgOPT.PGOPT['extlog'])
          if metacnt >= PgOPT.PGOPT['RSMAX']:
             metatotal += PgMeta.process_metadata("W", metacnt, PgOPT.PGOPT['emerol'])
             metacnt = 0
+         if PgLOG.PGLOG['DSCHECK'] and metacnt == 0:
+            PgCMD.set_dscheck_dcount(i, 0, PgOPT.PGOPT['extlog'])
          if 'EM' in PgOPT.params:
             PgLOG.PGLOG['PRGMSG'] = "{}/{}/{} of {} {} Web file record{} added/modified/processed".format(ADDCNT, MODCNT, i, ALLCNT, dsid, s)
       if not PgOPT.params['WF'][i]: continue

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/src/rda_python_dsarch/dsarch.usg

@@ -209,7 +209,8 @@ categories:
          [-(UD|UseDSARCH) UseDSARCHFlag]   # Y, P, W, I, or N
          [-(LC|Location) AccessLocationFlag]   # C - CGD, G - Glade, O - Object Store
          [-(BF|BackupFlag) QuasarBackupFlag]   # B - Backup, D - Backup & Drdata
-         [-(DF|DataFormat) DataFormat]     # BINARY, ASCII, NetCDF and etc.
+         [-(DA|AccessFlag) DataAccessFlags]    # B - Both, C - Complete, F - Faceted, or N - No
+         [-(DF|DataFormat) DataFormat]         # BINARY, ASCII, NetCDF and etc.
          [-(WH|WebHome) WebDataHomeDirectory]
          [-(WP|WebPath) WebFilePath]
          [-(SP|SavedPath) SavedFilePath]
@@ -279,28 +280,29 @@ categories:
 
   Valid dataset field names and their corresponding Info options:
 
-  Names  Info Options         Descriptions
-  T      -(TI|Title)          dataset title 
-  S      -(SP|SavedPath)      path relative to Saved home directory
-  H      -(WH|WebHome)        path for the web data home directory
-  W      -(WP|WebPath)        path relative to web home directory
-  F      -(DF|DataFormat)     default data content format
-  U      -(UD|UseDSARCH)      use DSARCH flag
-  L      -(LC|Location)       Online web file access location flags
-  Q      -(BF|BackupFlag)     Quasar Backup flags
-  V      -(GL|GroupLevel)     number of group levels to display
-  C      -(FL|FileLimit)      file count limit for separate group filelists
-  B      -(BL|ButtonLimit)    file count limit for showing script buttons
-  A      -(ML|MetaLink)       a link to content meta at the dataset level
-  W      -(NW|NoteWeb)        web data description
-  I      -(NI|NoteInternal)   internal data description
-  D      -(ND|NoteDocument)   document description
-  N      -(NS|NoteSoftware)   software description
-  G      -(GI|GroupIndex)     Group Indices to identify groups for periods
-  J      -(BD|BeginDate)      date data begin for dataset/group
-  K      -(BT|BeginTime)      time data begin for dataset/group
-  X      -(ED|EndDate)        date data end for dataset/group
-  Y      -(ET|EndTime)        time data end for dataset/group
+  Names  Info Options          Descriptions
+  T      -(TI|Title)           dataset title 
+  S      -(SP|SavedPath)       path relative to Saved home directory
+  H      -(WH|WebHome)         path for the web data home directory
+  W      -(WP|WebPath)         path relative to web home directory
+  F      -(DF|DataFormat)      default data content format
+  U      -(UD|UseDSARCH)       use DSARCH flag
+  L      -(LC|Location)        Online web file access location flags
+  Q      -(BF|BackupFlag)      Quasar Backup flags
+  A      -(DA|DataAccessFlag)  dataset data file list access flags
+  V      -(GL|GroupLevel)      number of group levels to display
+  C      -(FL|FileLimit)       file count limit for separate group filelists
+  B      -(BL|ButtonLimit)     file count limit for showing script buttons
+  M      -(ML|MetaLink)        a link to content meta at the dataset level
+  W      -(NW|NoteWeb)         web data description
+  I      -(NI|NoteInternal)    internal data description
+  D      -(ND|NoteDocument)    document description
+  N      -(NS|NoteSoftware)    software description
+  G      -(GI|GroupIndex)      Group Indices to identify groups for periods
+  J      -(BD|BeginDate)       date data begin for dataset/group
+  K      -(BT|BeginTime)       time data begin for dataset/group
+  X      -(ED|EndDate)         date data end for dataset/group
+  Y      -(ET|EndTime)         time data end for dataset/group
 
   Info option -OF (-OutputFile) is used to specify a file name to save the
   retrieved information for later usage. Result of this action is displayed
@@ -495,6 +497,7 @@ VersionIndex<:>InternalVersion<:>ExternalVersion<:>DOINumber<:>VersionStatus<:>B
          [-(GT|GroupType) GroupTypes]
          [-(GP|GroupPattern) FileNamePatterns]
          [-(BF|BackupFlag) QuasarBackupFlags]
+         [-(DA|AccessFlag) DataAccessFlags]    # B - Both, C - Complete, F - Faceted or N - No
          [-(SP|SavedPath) SavedFilePaths]   # relative to dataset saved home dir
          [-(WP|WebPath) WebFilePaths]       # relative to dataset data home dir
          [-(ML|MetaLink) MetadataLinks]
@@ -575,19 +578,20 @@ QSCAT observations.<:>
 
   Valid group field names and their corresponding Info options:
 
-  Names  Info Options         Descriptions
-  I      -(GI|GroupIndex)     group indices
-  G      -(GN|GroupName)      group names, IDs
-  X      -(PI|ParentIndex)    parent group indices
-  T      -(TI|Title)          group titles 
-  R      -(GT|GroupType)      group types
-  P      -(GP|GroupPattern)   common file name patterns for each group
-  Q      -(BF|BackupFlag)     Quasar Backup flags
-  S      -(SP|SavedPath)      path relative to saved home directory for groups
-  W      -(WP|WebPath)        path relative to web home directory for groups
-  A      -(ML|MetaLink)       a link to content meta at the group level
-  D      -(NW|NoteWeb)        group description of data on RDA Server
-  N      -(NI|NoteInternal)   group description of internal HPSS data
+  Names  Info Options          Descriptions
+  I      -(GI|GroupIndex)      group indices
+  G      -(GN|GroupName)       group names, IDs
+  X      -(PI|ParentIndex)     parent group indices
+  T      -(TI|Title)           group titles 
+  R      -(GT|GroupType)       group types
+  P      -(GP|GroupPattern)    common file name patterns for each group
+  Q      -(BF|BackupFlag)      Quasar Backup flags
+  A      -(DA|DataAccessFlag)  top group data file list access flags
+  S      -(SP|SavedPath)       path relative to saved home directory for groups
+  W      -(WP|WebPath)         path relative to web home directory for groups
+  M      -(ML|MetaLink)        a link to content meta at the group level
+  D      -(NW|NoteWeb)         group description of data on RDA Server
+  N      -(NI|NoteInternal)    group description of internal HPSS data
 
   Group information can be retrieved for specified groups per option -GI
   (-GroupIndex) or -GN (-GroupName), or per option -PI (-ParentIndex). Info
@@ -2173,6 +2177,10 @@ Information options are used to pass information, one or multiple values, into
   -BT or -BeginTime, for begin time of data files of a given group or the whole
   dataset, or begin time of a version control record.
 
+  -DA or -AcessFlag, (Alias: -DataAccessFlag, -FileListFlag). flags to show web file
+  lists at Dataset and/or top group levels; B-Both Filelists (default value),
+  C-Complete File List, F-Faceted Browse or N-No File List.
+
   -DB or -Debug, to turn on debug mode with specified information. This option
   provides up to 3 values, they are Debug Level, debug log file path and debug
   log file name. The debug level is mandatory for this option. It can be a

{rda_python_dsarch-1.0.14 → rda_python_dsarch-1.0.16}/src/rda_python_dsarch.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: rda_python_dsarch
-Version: 1.0.14
+Version: 1.0.16
 Summary: RDA Python package for utility program dsarch archive RDA data
 Author-email: Zaihua Ji <zji@ucar.edu>
 Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch