rda-python-dsarch 1.0.20__tar.gz → 1.0.22__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (18) hide show
  1. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/PKG-INFO +1 -1
  2. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/pyproject.toml +1 -1
  3. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch/PgArch.py +11 -7
  4. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch/dsarch.py +50 -25
  5. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch/dsarch.usg +22 -8
  6. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/PKG-INFO +1 -1
  7. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/LICENSE +0 -0
  8. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/MANIFEST.in +0 -0
  9. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/README.md +0 -0
  10. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/setup.cfg +0 -0
  11. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch/PgMeta.py +0 -0
  12. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch/__init__.py +0 -0
  13. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/SOURCES.txt +0 -0
  14. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/dependency_links.txt +0 -0
  15. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/entry_points.txt +0 -0
  16. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/requires.txt +0 -0
  17. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/src/rda_python_dsarch.egg-info/top_level.txt +0 -0
  18. {rda_python_dsarch-1.0.20 → rda_python_dsarch-1.0.22}/tests/test_dsarch.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rda_python_dsarch
3
- Version: 1.0.20
3
+ Version: 1.0.22
4
4
  Summary: RDA Python package for utility program dsarch archive RDA data
5
5
  Author-email: Zaihua Ji <zji@ucar.edu>
6
6
  Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "rda_python_dsarch"
7
- version = "1.0.20"
7
+ version = "1.0.22"
8
8
  authors = [
9
9
  { name="Zaihua Ji", email="zji@ucar.edu" },
10
10
  ]
@@ -151,6 +151,7 @@ PgOPT.OPTS = { # (!= 0) - setting actions
151
151
  'ET' : [2, 'EndTime', 32],
152
152
  'EV' : [2, 'ExternalVersion', 1],
153
153
  'FD' : [2, 'FileDate', 257],
154
+ 'FF' : [2, 'FileFlag', 1, 'FP'],
154
155
  'FS' : [2, 'FileStatus', 1, 'PI'],
155
156
  'FT' : [2, 'FileTime', 33],
156
157
  'GI' : [2, 'GroupIndex', 17],
@@ -196,7 +197,7 @@ PgOPT.OPTS = { # (!= 0) - setting actions
196
197
  }
197
198
 
198
199
  PgOPT.ALIAS = {
199
- 'AF' : ['FF', 'FileFormat', 'ExternalFormat'],
200
+ 'AF' : ['FileFormat', 'ExternalFormat'],
200
201
  'AL' : ['AsynchronousLimit'],
201
202
  'AO' : ['ActOption'],
202
203
  'AH' : ['ArchiveHelp', 'ArchHelp', 'ArchHelpFile'],
@@ -332,6 +333,7 @@ PgOPT.TBLHASH['wfile'] = {
332
333
  'Q' : ['QT', "btype", 1, "bfile.type"],
333
334
  'L' : ['LC', "locflag", 1],
334
335
  'H' : ['FS', "wfile.status", 1],
336
+ 'P' : ['FF', "fileflag", 1],
335
337
  'S' : ['SZ', "wfile.data_size", 0],
336
338
  'J' : ['FD', "wfile.date_modified", 1],
337
339
  'K' : ['FT', "wfile.time_modified", 1],
@@ -354,6 +356,7 @@ PgOPT.TBLHASH['sfile'] = {
354
356
  'Q' : ['QT', "btype", 1, "bfile.type"],
355
357
  'L' : ['LC', "locflag", 1],
356
358
  'H' : ['FS', "sfile.status", 1],
359
+ 'P' : ['FF', "fileflag", 1],
357
360
  'S' : ['SZ', "sfile.data_size", 0],
358
361
  'J' : ['FD', "sfile.date_modified", 1],
359
362
  'K' : ['FT', "sfile.time_modified", 1],
@@ -383,6 +386,7 @@ PgOPT.TBLHASH['hfile'] = {
383
386
  'N' : ['DF', "data_format", 1],
384
387
  'O' : ['DO', "disp_order", -1],
385
388
  'H' : ['FS', "status", 1],
389
+ 'P' : ['FF', "fileflag", 1],
386
390
  'S' : ['SZ', "data_size", 0],
387
391
  'R' : ['SR', "source", 1],
388
392
  'I' : ['ID', "init_date", 1],
@@ -411,10 +415,10 @@ PgOPT.PGOPT['MSET'] = "SA"
411
415
  PgOPT.PGOPT['dataset'] = "SWFULQA"
412
416
  PgOPT.PGOPT['dsperiod'] = "GJKXY"
413
417
  PgOPT.PGOPT['dsgroup'] = "IGXTPQASW"
414
- PgOPT.PGOPT['wfile'] = "FTIVMNLHS"
415
- PgOPT.PGOPT['sfile'] = "FTIVMNLHS"
418
+ PgOPT.PGOPT['wfile'] = "FTIVMNLHPS"
419
+ PgOPT.PGOPT['sfile'] = "FTIVMNLHPS"
416
420
  PgOPT.PGOPT['bfile'] = "FNMTHS"
417
- PgOPT.PGOPT['hfile'] = 'FMNLTHSU'
421
+ PgOPT.PGOPT['hfile'] = 'FMNLTHPSU'
418
422
  PgOPT.PGOPT['dsvrsn'] = "VIEDSJX"
419
423
  PgOPT.PGOPT['UACTS'] = (PgOPT.OPTS['AQ'][0]|PgOPT.OPTS['SD'][0]|PgOPT.OPTS['DL'][0])
420
424
 
@@ -422,10 +426,10 @@ PgOPT.PGOPT['UACTS'] = (PgOPT.OPTS['AQ'][0]|PgOPT.OPTS['SD'][0]|PgOPT.OPTS['DL']
422
426
  PgOPT.PGOPT['dsall'] = "TSHWFULQAVCBMPIDNGJKXYIJKXY" # include 'pdall'
423
427
  PgOPT.PGOPT['pdall'] = "GJKXY"
424
428
  PgOPT.PGOPT['gpall'] = "IGXTRQAPSWMDN"
425
- PgOPT.PGOPT['wfall'] = "FTCIXVMNOBQLHSJKAED"
426
- PgOPT.PGOPT['sfall'] = "FTCIXVMNOBQLHSJKD"
429
+ PgOPT.PGOPT['wfall'] = "FTCIXVMNOBQLHPSJKAED"
430
+ PgOPT.PGOPT['sfall'] = "FTCIXVMNOBQLHPSJKD"
427
431
  PgOPT.PGOPT['bfall'] = "FNMOTHSCJKD"
428
- PgOPT.PGOPT['hfall'] = 'FTLCMNOHSRIJKUD'
432
+ PgOPT.PGOPT['hfall'] = 'FTLCMNOHPSRIJKUD'
429
433
  PgOPT.PGOPT['vrsnall'] = "VIEDSJKXYN"
430
434
 
431
435
  # PgOPT.PGOPT['ofile'] reserved for original type 'B' hpps file name
@@ -444,7 +444,7 @@ def archive_web_files():
444
444
  vsnctl = 1 if pgrec and pgrec['vindex'] and pgrec['data_size'] else 0
445
445
  chksum = PgOPT.params['MC'][i]
446
446
  if warch and (vsnctl or not override):
447
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
447
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
448
448
  if info:
449
449
  if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
450
450
  PgLOG.pglog("Web-{}: Same-Checksum ARCHIVED at {}:{}".format(winfo, info['date_modified'], info['time_modified']), PgOPT.PGOPT['emllog'])
@@ -512,13 +512,13 @@ def archive_web_files():
512
512
  if not fnames: fnames = PgOPT.get_field_keys(tname, None, "G") # get setting fields if not yet
513
513
  info = None
514
514
  if warch:
515
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
515
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
516
516
  elif oarch:
517
517
  info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
518
518
  elif pgrec:
519
519
  info = get_file_origin_info(wfile, pgrec)
520
520
  elif locflag == 'O':
521
- info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
521
+ info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
522
522
  wid = set_one_webfile(i, pgrec, wfile, fnames, type, info)
523
523
  if not wid:
524
524
  PgOPT.params['LF'][i] = PgOPT.params['WF'][i] = None
@@ -674,7 +674,7 @@ def archive_help_files():
674
674
 
675
675
  chksum = PgOPT.params['MC'][i]
676
676
  if harch and not OVERRIDE:
677
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
677
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
678
678
  if info:
679
679
  if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
680
680
  PgLOG.pglog("Help-{}: Same-Checksum ARCHIVED at {}:{}".format(hinfo, info['date_modified'], info['time_modified']), PgOPT.PGOPT['emllog'])
@@ -732,7 +732,7 @@ def archive_help_files():
732
732
  if not fnames: fnames = PgOPT.get_field_keys(tname, None, "G") # get setting fields if not yet
733
733
  info = None
734
734
  if harch:
735
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
735
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
736
736
  elif oarch:
737
737
  info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
738
738
  elif pgrec:
@@ -895,7 +895,7 @@ def archive_saved_files():
895
895
  vsnctl = 1 if pgrec and pgrec['vindex'] and pgrec['data_size'] else 0
896
896
  chksum = PgOPT.params['MC'][i]
897
897
  if sarch and (vsnctl or not override):
898
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
898
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
899
899
  if info:
900
900
  if pgrec and chksum and pgrec['checksum'] and pgrec['checksum'] == chksum:
901
901
  PgLOG.pglog("Saved-{}: Same-Checksum ARCHIVED at {}:{}".format(sinfo, info['date_modified'], info['time_modified']), PgOPT.PGOPT['emllog'])
@@ -959,7 +959,7 @@ def archive_saved_files():
959
959
  if not fnames: fnames = PgOPT.get_field_keys(tname, None, "G") # get setting fields if not yet
960
960
  info = None
961
961
  if sarch:
962
- info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol'])
962
+ info = PgFile.check_local_file(afile, 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
963
963
  elif oarch:
964
964
  info = PgFile.check_object_file(ofile, bucket, 1, PgOPT.PGOPT['emerol'])
965
965
  elif pgrec:
@@ -1059,7 +1059,7 @@ def crosscopy_web_files(aname):
1059
1059
  ofile = PgLOG.join_paths(dsid, wfile)
1060
1060
  warch = oarch = 1
1061
1061
  PgLOG.pglog(winfo + ": Cross {} Web file ...".format(aname), PgLOG.WARNLG)
1062
- info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol'])
1062
+ info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
1063
1063
  if info:
1064
1064
  warch = 0
1065
1065
  elif info is not None:
@@ -1231,7 +1231,7 @@ def crosscopy_help_files(aname):
1231
1231
  if pgrec and PgOPT.params['HF'][i] != hfile: PgOPT.params['HF'][i] = hfile
1232
1232
  harch = oarch = 1
1233
1233
  PgLOG.pglog(hinfo + ": Cross {} Help file ...".format(aname), PgLOG.WARNLG)
1234
- info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol'])
1234
+ info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
1235
1235
  if info:
1236
1236
  harch = 0
1237
1237
  elif info is not None:
@@ -1385,7 +1385,7 @@ def crosscopy_saved_files(aname):
1385
1385
  if pgrec and PgOPT.params['SF'][i] != sfile: PgOPT.params['SF'][i] = sfile
1386
1386
  sarch = oarch = 1
1387
1387
  PgLOG.pglog(sinfo + ": Cross {} Saved file ...".format(aname), PgLOG.WARNLG)
1388
- info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol'])
1388
+ info = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
1389
1389
  if info:
1390
1390
  sarch = 0
1391
1391
  elif info is not None:
@@ -1673,7 +1673,7 @@ def tar_backup_savedfiles(tarfile, tinfo, ccnt, chkstat):
1673
1673
  else:
1674
1674
  tardir = "{}/{}".format(dshome, stype)
1675
1675
  savedfile = "{}/{}".format(tardir, sfile) if relative else sfile
1676
- if not op.isfile(savedfile):
1676
+ if not op.exists(savedfile):
1677
1677
  PgLOG.pglog(savedfile + ": Saved file not exists to backup", PgOPT.PGOPT['extlog'])
1678
1678
  if relative:
1679
1679
  tarcmds[i] = "tar -{}vf {} -C {} {}".format(topt, tarfile, tardir, sfile)
@@ -1761,7 +1761,7 @@ def tar_backup_webfiles(tarfile, tinfo, ccnt, chkstat):
1761
1761
  else:
1762
1762
  tardir = dshome
1763
1763
  webfile = "{}/{}".format(tardir, wfile) if relative else wfile
1764
- if not op.isfile(webfile):
1764
+ if not op.exists(webfile):
1765
1765
  PgLOG.pglog(webfile + ": Web file not exists to backup", PgOPT.PGOPT['extlog'])
1766
1766
  if relative:
1767
1767
  tarcmds[i] = "tar -{}vf {} -C {} {}".format(topt, tarfile, tardir, wfile)
@@ -1951,12 +1951,12 @@ def get_backup_member_file(pgrec, tarfile, tardir):
1951
1951
  tarcmd = "tar -xvf {} -C {} {}".format(tarfile, tardir, mfile)
1952
1952
  PgLOG.pgsystem(tarcmd, PgOPT.PGOPT['extlog'], 5)
1953
1953
  afile = '{}/{}'.format(tardir, mfile)
1954
- ainfo = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['extlog'])
1954
+ ainfo = PgFile.check_local_file(afile, 0, PgOPT.PGOPT['extlog']|PgLOG.PFSIZE)
1955
1955
 
1956
1956
  if ainfo and mfile != wfile:
1957
1957
  nfile = '{}/{}'.format(tardir, wfile)
1958
1958
  PgFile.move_local_file(nfile, afile, PgOPT.PGOPT['extlog'])
1959
- ainfo = PgFile.check_local_file(nfile, 0, PgOPT.PGOPT['extlog'])
1959
+ ainfo = PgFile.check_local_file(nfile, 0, PgOPT.PGOPT['extlog']|PgLOG.PFSIZE)
1960
1960
 
1961
1961
  return ainfo
1962
1962
 
@@ -2177,7 +2177,7 @@ def crosscopy_backup_files():
2177
2177
  #
2178
2178
  def validate_gladearch(file, ofile, i):
2179
2179
 
2180
- info = PgFile.check_local_file(file, 6, PgOPT.PGOPT['errlog'])
2180
+ info = PgFile.check_local_file(file, 6, PgOPT.PGOPT['errlog']|PgLOG.PFSIZE)
2181
2181
  if not info:
2182
2182
  PgLOG.pglog("Error archiving {} to {}".format(ofile, file), PgLOG.LGEREX)
2183
2183
  elif PgOPT.OPTS['SZ'][2]&2 and PgOPT.params['SZ'] and info['data_size'] != PgOPT.params['SZ'][i]:
@@ -2842,7 +2842,7 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
2842
2842
  record['uid'] = PgOPT.PGOPT['UID']
2843
2843
  if not (info and info['date_modified']):
2844
2844
  info = PgFile.check_local_file(PgArch.get_web_path(i, file, 1, type),
2845
- 1, PgOPT.PGOPT['emerol'])
2845
+ 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
2846
2846
  if info:
2847
2847
  record['data_size'] = info['data_size']
2848
2848
  record['date_modified'] = info['date_modified']
@@ -2869,6 +2869,8 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
2869
2869
  record['status'] = 'I'
2870
2870
  if 'checksum' in record: PgLOG.pglog("md5({}-{})={}".format(dsid, file, record['checksum']), PgLOG.LOGWRN)
2871
2871
  if 'gindex' in record and record['gindex']: record['tindex'] = PgMeta.get_top_gindex(dsid, record['gindex'])
2872
+ stat = check_file_flag(file, info, record, pgrec)
2873
+ if not stat: return stat
2872
2874
  if pgrec:
2873
2875
  if pgrec['wfile'] != file: record['wfile'] = file
2874
2876
  if 'bid' in record and record['bid'] and pgrec['bid'] and record['bid'] != pgrec['bid']:
@@ -2889,9 +2891,10 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
2889
2891
  record['uid'] = PgOPT.PGOPT['UID']
2890
2892
  if 'status' not in record: record['status'] = 'P'
2891
2893
  if not info:
2892
- info = PgFile.check_local_file(PgArch.get_web_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
2894
+ info = PgFile.check_local_file(PgArch.get_web_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
2895
+ stat = check_file_flag(file, info, record)
2896
+ if not stat: return stat
2893
2897
  if info:
2894
- if not info['isfile']: return PgLOG.pglog("file: is a directory", PgOPT.PGOPT['emlerr'])
2895
2898
  record['data_size'] = info['data_size']
2896
2899
  record['date_modified'] = info['date_modified']
2897
2900
  record['time_modified'] = info['time_modified']
@@ -2912,6 +2915,22 @@ def set_one_webfile(i, pgrec, file, flds, type, info = None, ndsid = None, sact
2912
2915
 
2913
2916
  return wid
2914
2917
 
2918
+ #
2919
+ # check
2920
+ #
2921
+ def check_file_flag(file, info, record, pgrec = None):
2922
+
2923
+ if not info: return PgLOG.SUCCESS
2924
+
2925
+ fflag = 'F' if info['isfile'] else 'P'
2926
+ if 'fileflag' in record:
2927
+ if fflag != record['fileflag']:
2928
+ return PgLOG.pglog("{}: Cannot set File Flag '{}' to '{}'".format(file, fflag, record['fileflag']), PgOPT.PGOPT['emlerr'])
2929
+ elif not (pgrec and pgrec['fileflag'] == fflag):
2930
+ record['fileflag'] = fflag
2931
+
2932
+ return PgLOG.SUCCESS
2933
+
2915
2934
  #
2916
2935
  # add or modify Help file information
2917
2936
  #
@@ -2992,7 +3011,7 @@ def set_one_helpfile(i, pgrec, file, flds, type, info = None, ndsid = None):
2992
3011
  if pgrec and (pgrec['status'] == 'D' or PgOPT.PGOPT['ACTS']&PgOPT.OPTS['AH'][0]):
2993
3012
  record['uid'] = PgOPT.PGOPT['UID']
2994
3013
  if not (info and info['date_modified']):
2995
- info = PgFile.check_local_file(PgArch.get_help_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
3014
+ info = PgFile.check_local_file(PgArch.get_help_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
2996
3015
  if info:
2997
3016
  record['data_size'] = info['data_size']
2998
3017
  record['date_modified'] = info['date_modified']
@@ -3010,6 +3029,8 @@ def set_one_helpfile(i, pgrec, file, flds, type, info = None, ndsid = None):
3010
3029
  if 'url' in record:
3011
3030
  record['locflag'] = 'R'
3012
3031
  record['date_modified'] = PgUtil.curdate()
3032
+ stat = check_file_flag(file, info, record, pgrec)
3033
+ if not stat: return stat
3013
3034
  if pgrec:
3014
3035
  if pgrec['hfile'] != file: record['hfile'] = file
3015
3036
  if not ndsid:
@@ -3029,9 +3050,10 @@ def set_one_helpfile(i, pgrec, file, flds, type, info = None, ndsid = None):
3029
3050
  record['uid'] = PgOPT.PGOPT['UID']
3030
3051
  if 'status' not in record: record['status'] = 'P'
3031
3052
  if not (info or 'url' in record):
3032
- info = PgFile.check_local_file(PgArch.get_help_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
3053
+ info = PgFile.check_local_file(PgArch.get_help_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
3054
+ stat = check_file_flag(file, info, record, pgrec)
3055
+ if not stat: return stat
3033
3056
  if info:
3034
- if not info['isfile']: return PgLOG.pglog("file: is a directory", PgOPT.PGOPT['emlerr'])
3035
3057
  record['data_size'] = info['data_size']
3036
3058
  record['date_modified'] = info['date_modified']
3037
3059
  record['time_modified'] = info['time_modified']
@@ -3164,7 +3186,7 @@ def set_one_savedfile(i, pgrec, file, flds, type, info = None, ndsid = None, sac
3164
3186
  if pgrec and PgOPT.PGOPT['ACTS']&PgOPT.OPTS['AS'][0]:
3165
3187
  record['uid'] = PgOPT.PGOPT['UID']
3166
3188
  if not (info and info['date_modified']):
3167
- info = PgFile.check_local_file(PgArch.get_saved_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
3189
+ info = PgFile.check_local_file(PgArch.get_saved_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
3168
3190
  if info:
3169
3191
  record['data_size'] = info['data_size']
3170
3192
  record['date_modified'] = info['date_modified']
@@ -3188,6 +3210,8 @@ def set_one_savedfile(i, pgrec, file, flds, type, info = None, ndsid = None, sac
3188
3210
  record['status'] = 'I'
3189
3211
  if 'checksum' in record and record['checksum']: PgLOG.pglog("md5({}-{})={}".format(dsid, file, record['checksum']), PgLOG.LOGWRN)
3190
3212
  if 'gindex' in record and record['gindex']: record['tindex'] = PgMeta.get_top_gindex(dsid, record['gindex'])
3213
+ stat = check_file_flag(file, info, record, pgrec)
3214
+ if not stat: return stat
3191
3215
  if pgrec:
3192
3216
  if pgrec['sfile'] != file: record['sfile'] = file
3193
3217
  if not ndsid:
@@ -3212,9 +3236,10 @@ def set_one_savedfile(i, pgrec, file, flds, type, info = None, ndsid = None, sac
3212
3236
  record['uid'] = PgOPT.PGOPT['UID']
3213
3237
  if 'status' not in record: record['status'] = 'P'
3214
3238
  if not info:
3215
- info = PgFile.check_local_file(PgArch.get_saved_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol'])
3239
+ info = PgFile.check_local_file(PgArch.get_saved_path(i, file, 1, type), 1, PgOPT.PGOPT['emerol']|PgLOG.PFSIZE)
3240
+ stat = check_file_flag(file, info, record, pgrec)
3241
+ if not stat: return stat
3216
3242
  if info:
3217
- if not info['isfile']: return PgLOG.pglog(file + ": is a directory", PgOPT.PGOPT['emlerr'])
3218
3243
  record['data_size'] = info['data_size']
3219
3244
  record['date_modified'] = info['date_modified']
3220
3245
  record['time_modified'] = info['time_modified']
@@ -4445,7 +4470,7 @@ def web_to_saved_files():
4445
4470
  #
4446
4471
  def get_file_origin_info(fname, pgrec):
4447
4472
 
4448
- info = {'isfile' : 1, 'data_size' : pgrec['data_size']}
4473
+ info = {'isfile' : (0 if 'fileflag' in pgrec and pgrec['fileflag'] == 'P' else 1), 'data_size' : pgrec['data_size']}
4449
4474
  info['fname'] = op.basename(fname)
4450
4475
  info['date_modified'] = pgrec['date_modified']
4451
4476
  info['time_modified'] = pgrec['time_modified']
@@ -734,6 +734,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
734
734
  [-(PO|PatternOffset) PatternStringOffset]
735
735
  [-(ST|SavedFileType) SavedFileTypes] # Primary, VersionControl, Work, oRiginal
736
736
  [-(FS|FileStatus) SavedFileStatus] # Public or Interal file
737
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
737
738
  [-(LC|Location) StorageLocationFlags] # CGD data; Glade, Object or Both
738
739
  [-(DF|DataFormat) DataFormats] # BINARY, ASCII, NetCDF and etc.
739
740
  [-(AF|ArchiveFormat) FileArchiveFormats] # tarred, Compressed and etc.
@@ -796,6 +797,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
796
797
  [-(SF|SavedFile) SavedFileNames]
797
798
  [-(ST|SavedFileType) SavedFileTypes]
798
799
  [-(FS|FileStatus) SavedFileStatus]
800
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
799
801
  [-(LC|Location) StorageLocationFlags]
800
802
  [-(GI|GroupIndex) GroupIndices]
801
803
  [-(GN|GroupName) GroupNames]
@@ -831,6 +833,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
831
833
  Q -(QT|QuasarType) Quasar backyp file types
832
834
  L -(LC|Location) Saved file storage location flag
833
835
  H -(FS|FileStatus) Saved file archiving status
836
+ P -(FF|FileFlag) File type flag, File or Path
834
837
  S -(SZ|Size) Saved file sizes
835
838
  J -(FD|FileDate) date data last modified on
836
839
  K -(FT|FileTime) time data last modified at
@@ -866,6 +869,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
866
869
  [-(WT|WebFileType) WebFileTypes] # Web Data or NCAR Data
867
870
  [-(LC|Location) StorageLocationFlags]
868
871
  [-(FS|FileStatus) WebFileStatus] # Public or interal file
872
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
869
873
  [-(DF|DataFormat) DataFormats] # BINARY, ASCII, NetCDF and etc.
870
874
  [-(AF|ArchiveFormat) FileArchiveFormats] # tarred, Compressed and etc.
871
875
  [-(DO|DisplayOrder] DisplayOrderIndices]
@@ -940,6 +944,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
940
944
  [-(WF|WebFile) WebFileNames]
941
945
  [-(WT|WebFileType) WebFileTypes]
942
946
  [-(FS|FileStatus) WebFileStatus]
947
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
943
948
  [-(LC|Location) StorageLocationFlags]
944
949
  [-(GI|GroupIndex) GroupIndices]
945
950
  [-(GN|GroupName) GroupNames]
@@ -975,6 +980,7 @@ d260001-G2: DS-552780725 DC-522 WC-522 SS-465789806 SC-51
975
980
  Q -(QT|QuasarType) Quasar backyp fie types
976
981
  L -(LC|Location) Web file storage location flag
977
982
  H -(FS|FileStatus) Web file archiving status
983
+ P -(FF|FileFlag) File type flag, File or Path
978
984
  S -(SZ|Size) Web file sizes
979
985
  J -(FD|FileDate) date data last modified on
980
986
  K -(FT|FileTime) time data last modified at
@@ -1021,10 +1027,11 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1021
1027
  [-(SR|Source) ContributionSource]
1022
1028
  [-(ON|OrderNames) OrderNameString]
1023
1029
  [-(OB|OrderBy) OrderByPattern]
1024
- [-(HT|HelpFileType) HelpFileTypes] # Document or Software
1025
- [-(FS|FileStatus) HelpFileStatus] # Public or Interal file
1030
+ [-(HT|HelpFileType) HelpFileTypes] # Document or Software
1031
+ [-(FS|FileStatus) HelpFileStatus] # Public or Interal file
1032
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
1026
1033
  [-(LC|Location) StorageLocationFlags] # Glade, Object, Both or Remote
1027
- [-(DF|DataFormat) DataFormats] # Python, PDF, and etc.
1034
+ [-(DF|DataFormat) DataFormats] # Python, PDF, and etc.
1028
1035
  [-(AF|ArchiveFormat) FileArchiveFormats] # tarred, Compressed and etc.
1029
1036
  [-(DO|DisplayOrder] DisplayOrderIndices]
1030
1037
  [-(SZ|Size) FileSizes]
@@ -1075,6 +1082,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1075
1082
  [-(HF|HelpFile) HelpFileNames]
1076
1083
  [-(HT|HelpFileType) HelpFileTypes] # Document or Software
1077
1084
  [-(FS|FileStatus) SavedFileStatus]
1085
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
1078
1086
  [-(LC|Location) StorageLocationFlags]
1079
1087
  [-(SR|Source) ContributionSource]
1080
1088
  [-(DE|Description) SavedFileDescriptions]
@@ -1098,6 +1106,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1098
1106
  O -(DO|DisplayOrder) display order indices
1099
1107
  T -(HT|HelpFileType) Help file archiving types
1100
1108
  H -(FS|FileStatus) Help file archiving status
1109
+ P -(FF|FileFlag) File type flag, File or Path
1101
1110
  L -(LC|Location) Help file storage location flag
1102
1111
  S -(SZ|Size) Help file sizes
1103
1112
  R -(SR|Source) Help file contribution source
@@ -1235,6 +1244,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1235
1244
  [-(PO|PatternOffset) PatternStringOffset]
1236
1245
  [-(ST|SavedFileType) SavedFileTypes]
1237
1246
  [-(FS|FileStatus) SavedFileStatus] # P-Public or I-Internal
1247
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
1238
1248
  [-(LC|Location) StorageLocationFlags]
1239
1249
  [-(DF|DataFormat) DataFormat] # BINARY, ASCII, NetCDF and etc.
1240
1250
  [-(AF|ArchiveFormat) FileArchiveFormat] # tarred, compressed and etc.
@@ -1334,16 +1344,17 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1334
1344
  dsarch [-(DS|dataset)] dsnnn.n -(AW|ArchiveWebFile) [Mode Options]
1335
1345
  [-(LF|LocalFile) LocalFileNames] # mandatory unless -LD or -LL present
1336
1346
  [-(LD|LocalDirectory) LocalDirectoryName] # directory for local file names
1337
- [-(LL|LocalFileList) LocalFileListName] # contains a list of local file names
1338
- [-(WF|WebFile) webFileNames] # default to -LF
1347
+ [-(LL|LocalFileList) LocalFileListName] # contains a list of local file names
1348
+ [-(WF|WebFile) webFileNames] # default to -LF
1339
1349
  [-(GI|GroupIndex) GroupIndices]
1340
1350
  [-(TG|TopGroupIndex) TopGroupIndices]
1341
1351
  [-(GN|GroupName) GroupNames]
1342
1352
  [-(PO|PatternOffset) PatternStringOffset]
1343
- [-(WT|WebFileType) WebFileTypes] # Data, Ncar Data
1344
- [-(FS|FileStatus) WebFileStatus] # P-Public or I-Internal
1353
+ [-(WT|WebFileType) WebFileTypes] # Data, Ncar Data
1354
+ [-(FS|FileStatus) WebFileStatus] # P-Public or I-Internal
1355
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
1345
1356
  [-(LC|Location) StorageLocationFlags]
1346
- [-(DF|DataFormat) DataFormat] # BINARY, ASCII, NetCDF and etc.
1357
+ [-(DF|DataFormat) DataFormat] # BINARY, ASCII, NetCDF and etc.
1347
1358
  [-(AF|ArchiveFormat) FileArchiveFormat] # tarred, compressed and etc.
1348
1359
  [-(DO|DisplayOrder] DisplayOrderIndices]
1349
1360
  [-(SZ|Size) FileSizes]
@@ -1449,6 +1460,7 @@ IMMA_2.4_2007.tar<:>37<:>IMMA<:>Z.TAR<:>D<:>656820224<:>
1449
1460
  [-(HF|HelpFile) HelpFileNames] # default to -LF
1450
1461
  [-(HT|HelpFileType) HelpFileTypes]
1451
1462
  [-(FS|FileStatus) HelpFileStatus] # P-Public or I-Internal
1463
+ [-(FF|FileFlag) FilesOrPaths] # file type flag, File or Path
1452
1464
  [-(DF|DataFormat) FileContentFormat] # PDF, Python and etc.
1453
1465
  [-(AF|ArchiveFormat) FileArchiveFormat] # tarred, compressed and etc.
1454
1466
  [-(DO|DisplayOrder] DisplayOrderIndices]
@@ -2222,6 +2234,8 @@ Information options are used to pass information, one or multiple values, into
2222
2234
 
2223
2235
  -FD or -FileDate, for date of a data file last modified on.
2224
2236
 
2237
+ -FF or -FileFlag, File type flag, F for File and P for Path.
2238
+
2225
2239
  -FS or -FileStatus, File status, a single letter status flag, P - show file
2226
2240
  to public users; and I - to show file to specialist only.
2227
2241
 
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: rda_python_dsarch
3
- Version: 1.0.20
3
+ Version: 1.0.22
4
4
  Summary: RDA Python package for utility program dsarch archive RDA data
5
5
  Author-email: Zaihua Ji <zji@ucar.edu>
6
6
  Project-URL: Homepage, https://github.com/NCAR/rda-python-dsarch