shancx 1.8.92__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (166) hide show
  1. shancx/3D/__init__.py +25 -0
  2. shancx/Algo/Class.py +11 -0
  3. shancx/Algo/CudaPrefetcher1.py +112 -0
  4. shancx/Algo/Fake_image.py +24 -0
  5. shancx/Algo/Hsml.py +391 -0
  6. shancx/Algo/L2Loss.py +10 -0
  7. shancx/Algo/MetricTracker.py +132 -0
  8. shancx/Algo/Normalize.py +66 -0
  9. shancx/Algo/OptimizerWScheduler.py +38 -0
  10. shancx/Algo/Rmageresize.py +79 -0
  11. shancx/Algo/Savemodel.py +33 -0
  12. shancx/Algo/SmoothL1_losses.py +27 -0
  13. shancx/Algo/Tqdm.py +62 -0
  14. shancx/Algo/__init__.py +121 -0
  15. shancx/Algo/checknan.py +28 -0
  16. shancx/Algo/iouJU.py +83 -0
  17. shancx/Algo/mask.py +25 -0
  18. shancx/Algo/psnr.py +9 -0
  19. shancx/Algo/ssim.py +70 -0
  20. shancx/Algo/structural_similarity.py +308 -0
  21. shancx/Algo/tool.py +704 -0
  22. shancx/Calmetrics/__init__.py +97 -0
  23. shancx/Calmetrics/calmetrics.py +14 -0
  24. shancx/Calmetrics/calmetricsmatrixLib.py +147 -0
  25. shancx/Calmetrics/rmseR2score.py +35 -0
  26. shancx/Clip/__init__.py +50 -0
  27. shancx/Cmd.py +126 -0
  28. shancx/Config_.py +26 -0
  29. shancx/Df/DataFrame.py +11 -2
  30. shancx/Df/__init__.py +17 -0
  31. shancx/Df/tool.py +0 -0
  32. shancx/Diffm/Psamples.py +18 -0
  33. shancx/Diffm/__init__.py +0 -0
  34. shancx/Diffm/test.py +207 -0
  35. shancx/Doc/__init__.py +214 -0
  36. shancx/E/__init__.py +178 -152
  37. shancx/Fillmiss/__init__.py +0 -0
  38. shancx/Fillmiss/imgidwJU.py +46 -0
  39. shancx/Fillmiss/imgidwLatLonJU.py +82 -0
  40. shancx/Gpu/__init__.py +55 -0
  41. shancx/H9/__init__.py +126 -0
  42. shancx/H9/ahi_read_hsd.py +877 -0
  43. shancx/H9/ahisearchtable.py +298 -0
  44. shancx/H9/geometry.py +2439 -0
  45. shancx/Hug/__init__.py +81 -0
  46. shancx/Inst.py +22 -0
  47. shancx/Lib.py +31 -0
  48. shancx/Mos/__init__.py +37 -0
  49. shancx/NN/__init__.py +235 -106
  50. shancx/Path1.py +161 -0
  51. shancx/Plot/GlobMap.py +276 -116
  52. shancx/Plot/__init__.py +491 -1
  53. shancx/Plot/draw_day_CR_PNG.py +4 -21
  54. shancx/Plot/exam.py +116 -0
  55. shancx/Plot/plotGlobal.py +325 -0
  56. shancx/{radar_nmc.py → Plot/radarNmc.py} +4 -34
  57. shancx/{subplots_single_china_map.py → Plot/single_china_map.py} +1 -1
  58. shancx/Point.py +46 -0
  59. shancx/QC.py +223 -0
  60. shancx/RdPzl/__init__.py +32 -0
  61. shancx/Read.py +72 -0
  62. shancx/Resize.py +79 -0
  63. shancx/SN/__init__.py +62 -123
  64. shancx/Time/GetTime.py +9 -3
  65. shancx/Time/__init__.py +66 -1
  66. shancx/Time/timeCycle.py +302 -0
  67. shancx/Time/tool.py +0 -0
  68. shancx/Train/__init__.py +74 -0
  69. shancx/Train/makelist.py +187 -0
  70. shancx/Train/multiGpu.py +27 -0
  71. shancx/Train/prepare.py +161 -0
  72. shancx/Train/renet50.py +157 -0
  73. shancx/ZR.py +12 -0
  74. shancx/__init__.py +333 -262
  75. shancx/args.py +27 -0
  76. shancx/bak.py +768 -0
  77. shancx/df2database.py +62 -2
  78. shancx/geosProj.py +80 -0
  79. shancx/info.py +38 -0
  80. shancx/netdfJU.py +231 -0
  81. shancx/sendM.py +59 -0
  82. shancx/tensBoard/__init__.py +28 -0
  83. shancx/wait.py +246 -0
  84. {shancx-1.8.92.dist-info → shancx-1.9.33.218.dist-info}/METADATA +15 -5
  85. shancx-1.9.33.218.dist-info/RECORD +91 -0
  86. {shancx-1.8.92.dist-info → shancx-1.9.33.218.dist-info}/WHEEL +1 -1
  87. my_timer_decorator/__init__.py +0 -10
  88. shancx/Dsalgor/__init__.py +0 -19
  89. shancx/E/DFGRRIB.py +0 -30
  90. shancx/EN/DFGRRIB.py +0 -30
  91. shancx/EN/__init__.py +0 -148
  92. shancx/FileRead.py +0 -44
  93. shancx/Gray2RGB.py +0 -86
  94. shancx/M/__init__.py +0 -137
  95. shancx/MN/__init__.py +0 -133
  96. shancx/N/__init__.py +0 -131
  97. shancx/Plot/draw_day_CR_PNGUS.py +0 -206
  98. shancx/Plot/draw_day_CR_SVG.py +0 -275
  99. shancx/Plot/draw_day_pre_PNGUS.py +0 -205
  100. shancx/Plot/glob_nation_map.py +0 -116
  101. shancx/Plot/radar_nmc.py +0 -61
  102. shancx/Plot/radar_nmc_china_map_compare1.py +0 -50
  103. shancx/Plot/radar_nmc_china_map_f.py +0 -121
  104. shancx/Plot/radar_nmc_us_map_f.py +0 -128
  105. shancx/Plot/subplots_compare_devlop.py +0 -36
  106. shancx/Plot/subplots_single_china_map.py +0 -45
  107. shancx/S/__init__.py +0 -138
  108. shancx/W/__init__.py +0 -132
  109. shancx/WN/__init__.py +0 -132
  110. shancx/code.py +0 -331
  111. shancx/draw_day_CR_PNG.py +0 -200
  112. shancx/draw_day_CR_PNGUS.py +0 -206
  113. shancx/draw_day_CR_SVG.py +0 -275
  114. shancx/draw_day_pre_PNGUS.py +0 -205
  115. shancx/makenetCDFN.py +0 -42
  116. shancx/mkIMGSCX.py +0 -92
  117. shancx/netCDF.py +0 -130
  118. shancx/radar_nmc_china_map_compare1.py +0 -50
  119. shancx/radar_nmc_china_map_f.py +0 -125
  120. shancx/radar_nmc_us_map_f.py +0 -67
  121. shancx/subplots_compare_devlop.py +0 -36
  122. shancx/tool.py +0 -18
  123. shancx/user/H8mess.py +0 -317
  124. shancx/user/__init__.py +0 -137
  125. shancx/user/cinradHJN.py +0 -496
  126. shancx/user/examMeso.py +0 -293
  127. shancx/user/hjnDAAS.py +0 -26
  128. shancx/user/hjnFTP.py +0 -81
  129. shancx/user/hjnGIS.py +0 -320
  130. shancx/user/hjnGPU.py +0 -21
  131. shancx/user/hjnIDW.py +0 -68
  132. shancx/user/hjnKDTree.py +0 -75
  133. shancx/user/hjnLAPSTransform.py +0 -47
  134. shancx/user/hjnMiscellaneous.py +0 -182
  135. shancx/user/hjnProj.py +0 -162
  136. shancx/user/inotify.py +0 -41
  137. shancx/user/matplotlibMess.py +0 -87
  138. shancx/user/mkNCHJN.py +0 -623
  139. shancx/user/newTypeRadar.py +0 -492
  140. shancx/user/test.py +0 -6
  141. shancx/user/tlogP.py +0 -129
  142. shancx/util_log.py +0 -33
  143. shancx/wtx/H8mess.py +0 -315
  144. shancx/wtx/__init__.py +0 -151
  145. shancx/wtx/cinradHJN.py +0 -496
  146. shancx/wtx/colormap.py +0 -64
  147. shancx/wtx/examMeso.py +0 -298
  148. shancx/wtx/hjnDAAS.py +0 -26
  149. shancx/wtx/hjnFTP.py +0 -81
  150. shancx/wtx/hjnGIS.py +0 -330
  151. shancx/wtx/hjnGPU.py +0 -21
  152. shancx/wtx/hjnIDW.py +0 -68
  153. shancx/wtx/hjnKDTree.py +0 -75
  154. shancx/wtx/hjnLAPSTransform.py +0 -47
  155. shancx/wtx/hjnLog.py +0 -78
  156. shancx/wtx/hjnMiscellaneous.py +0 -201
  157. shancx/wtx/hjnProj.py +0 -161
  158. shancx/wtx/inotify.py +0 -41
  159. shancx/wtx/matplotlibMess.py +0 -87
  160. shancx/wtx/mkNCHJN.py +0 -613
  161. shancx/wtx/newTypeRadar.py +0 -492
  162. shancx/wtx/test.py +0 -6
  163. shancx/wtx/tlogP.py +0 -129
  164. shancx-1.8.92.dist-info/RECORD +0 -99
  165. /shancx/{Dsalgor → Algo}/dsalgor.py +0 -0
  166. {shancx-1.8.92.dist-info → shancx-1.9.33.218.dist-info}/top_level.txt +0 -0
shancx/Gray2RGB.py DELETED
@@ -1,86 +0,0 @@
1
-
2
- import cv2
3
- import numpy as np
4
-
5
- def main():
6
- with open('./gsmap_nrt.20241023.0300.dat', 'rb') as f:
7
- img = np.frombuffer(f.read(), np.uint8)
8
-
9
- print(f"Array size: {img.size}") # 输出数组大小
10
-
11
- channels = 3
12
- total_pixels = img.size // channels # 总像素数
13
-
14
- # 尝试不同的宽度,找到合适的高度
15
- for width in [5400, 4800, 640]: # 可以尝试不同的宽度
16
- if total_pixels % width == 0:
17
- height = total_pixels // width
18
- try:
19
- # 转换为三维数组
20
- img_3d = img.reshape((height, width, channels))
21
-
22
- # 确保数据类型为 uint8
23
- img_3d = img_3d.astype(np.uint8)
24
-
25
- # 保存图像
26
- cv2.imwrite('./img.png', img_3d)
27
- print(f"Successfully reshaped to {height}x{width}x{channels} and saved as img.png")
28
- break # 成功重塑后退出循环
29
- except ValueError:
30
- print(f"Failed to reshape with width {width}.")
31
- else:
32
- print("Could not find a suitable width to reshape the array.")
33
-
34
- # if __name__ == '__main__':
35
- # main()
36
-
37
-
38
- from matplotlib.colors import ListedColormap
39
- from PIL import Image
40
- import matplotlib.pyplot as plt
41
- import datetime
42
- from shancx import crDir
43
- import os
44
- from hjnwtx.colormap import cmp_hjnwtx
45
- import numpy as np
46
-
47
- cmp = {}
48
- newcolorsAU = np.array([
49
- [245,245,255,255],
50
- [180,180,255,255],
51
- [120,120,255,255],
52
- [20,20,255, 255],
53
- [0,216,195, 255],
54
- [0,150,144, 255],
55
- [0,102,102,255],
56
- [255,255,0,255],
57
- [255,200,0,255],
58
- [255,150,0,255],
59
- [255,100,0,255],
60
- [255,0,0,255],
61
- [200,0,0,255],
62
- [120,0,0,255],
63
- [40,0,0,255]])/255
64
- cmp["radar_AU"] = ListedColormap(newcolorsAU)
65
-
66
-
67
- def drawimg(array_dt):
68
- now_str = datetime.datetime.now().strftime("%Y%m%d%H%M%S")
69
- if len(array_dt.shape)==2:
70
- plt.imshow(array_dt,vmin=0,vmax=75,cmap=cmp_hjnwtx["radar_nmc"])
71
- plt.colorbar()
72
- outpath = f"./temp_RGB_{now_str}.png"
73
- crDir(outpath)
74
- plt.savefig(outpath)
75
- plt.close()
76
-
77
- def PlotRardar(path):
78
- imgMap = Image.open(path)
79
- imgMap = imgMap.convert('L')
80
- # 转换为 NumPy 数组
81
- # imgS_array = np.array(imgSource)
82
- imgM_array = np.array(imgMap)
83
- imgM_arrayf = np.where(imgM_array>0,imgM_array,np.nan)
84
- drawimg(imgM_arrayf)
85
-
86
- PlotRardar("./img.png")
shancx/M/__init__.py DELETED
@@ -1,137 +0,0 @@
1
- #!/usr/bin/python
2
- # -*- coding: utf-8 -*-
3
- import os
4
- def start():
5
- print("import successful")
6
- # constants
7
-
8
-
9
- __author__ = 'shancx'
10
-
11
- __author_email__ = 'shancx@126.com'
12
-
13
-
14
-
15
- # @Time : 2023/09/27 下午8:52
16
- # @Author : shanchangxi
17
- # @File : util_log.py
18
- import time
19
- import logging
20
- from logging import handlers
21
-
22
- def mkDir(path):
23
- if "." in path:
24
- os.makedirs(os.path.dirname(path),exist_ok=True)
25
- else:
26
- os.makedirs(path, exist_ok=True)
27
-
28
- loggers = logging.getLogger()
29
- loggers.setLevel(logging.INFO)
30
- log_name = '/home/scx/logs/project_M.log'
31
- mkDir(log_name)
32
- logfile = log_name
33
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
34
- time_rotating_file_handler.setLevel(logging.INFO)
35
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
36
- time_rotating_file_handler.setFormatter(formatter)
37
- loggers.addHandler(time_rotating_file_handler)
38
-
39
- """
40
- import tracback
41
- try:
42
-
43
- except Exception as e:
44
-
45
- logger.info(traceback.format_exc())
46
- """
47
-
48
- '''
49
- ##定義一個streamHandler
50
- # print_handler = logging.StreamHandler()
51
- # print_handler.setFormatter(formatter)
52
- # loggers.addHandler(print_handler)
53
- '''
54
-
55
-
56
- """
57
- from main import makeAll,options
58
- from multiprocessing import Pool
59
- import datetime
60
- from config import logger,output
61
- import time
62
- import pandas as pd
63
- import os
64
- from itertools import product
65
- import threading
66
-
67
- def excuteCommand(cmd):
68
- print(cmd)
69
- os.system(cmd)
70
-
71
- def gpuPro(makeListUTC, isPhase, isDebug, gpu, isOverwrite):
72
- productList = product(makeListUTC, [isPhase], [isDebug], [gpu], [isOverwrite])
73
-
74
- with Pool(4) as p:
75
- p.map(makeAll, productList)
76
-
77
- if __name__ == '__main__':
78
- cfg = options()
79
- isPhase = cfg.isPhase
80
- isDebug = cfg.isDebug
81
- sepSec = cfg.sepSec
82
- gpu = cfg.gpu
83
- pool = cfg.pool
84
- isOverwrite = cfg.isOverwrite
85
- timeList = pd.date_range(cfg.times[0], cfg.times[-1], freq=f"{sepSec}s")
86
- logger.info(f"时间段check {timeList}")
87
- gpuNum = 2
88
- eachGPU = 4
89
-
90
- makeListUTC = []
91
- for UTC in timeList:
92
- UTCStr = UTC.strftime("%Y%m%d%H%M")
93
- outpath = f"{output}/{UTCStr[:4]}/{UTCStr[:8]}/MSP2_WTX_AIW_QPF_L88_CHN_{UTCStr}_00000-00300-00006.nc"
94
- if not os.path.exists(outpath) or isOverwrite:
95
- makeListUTC.append(UTC)
96
- [print(element) for element in makeListUTC]
97
-
98
- phaseCMD = "--isPhase" if isPhase else ""
99
- debugCMD = "--isDebug" if isDebug else ""
100
- OverwriteCMD = "--isOverwrite"
101
- gpuCMD = f"--gpu={gpu}"
102
- # cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {OverwriteCMD} {gpuCMD}",makeListUTC))
103
- cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {gpuCMD}",makeListUTC))
104
-
105
- with Pool(pool) as p:
106
- p.map(excuteCommand, cmdList)
107
- """
108
-
109
-
110
- '''
111
- # @Time : 2023/09/27 下午8:52
112
- # @Author : shanchangxi
113
- # @File : util_log.py
114
- import time
115
- import logging
116
- from logging import handlers
117
-
118
- logger = logging.getLogger()
119
- logger.setLevel(logging.INFO)
120
- log_name = 'project_tim_tor.log'
121
- logfile = log_name
122
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
123
- time_rotating_file_handler.setLevel(logging.INFO)
124
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
125
- time_rotating_file_handler.setFormatter(formatter)
126
- logger.addHandler(time_rotating_file_handler)
127
- print_handler = logging.StreamHandler()
128
- print_handler.setFormatter(formatter)
129
- logger.addHandler(print_handler)
130
-
131
- '''
132
-
133
- '''
134
- ###解决方法 pip install torch==2.4.0 torchvision torchaudio三个同时安装 python 3.12 解决cuda启动不了的问题
135
-
136
- Res网络
137
- '''
shancx/MN/__init__.py DELETED
@@ -1,133 +0,0 @@
1
- #!/usr/bin/python
2
- # -*- coding: utf-8 -*-
3
- import os
4
- def start():
5
- print("import successful")
6
- # constants
7
- __author__ = 'shancx'
8
-
9
- __author_email__ = 'shancx@126.com'
10
-
11
- # @Time : 2023/09/27 下午8:52
12
- # @Author : shanchangxi
13
- # @File : util_log.py
14
- import time
15
- import logging
16
- from logging import handlers
17
-
18
- def mkDir(path):
19
- if "." in path:
20
- os.makedirs(os.path.dirname(path),exist_ok=True)
21
- else:
22
- os.makedirs(path, exist_ok=True)
23
-
24
- loggers = logging.getLogger()
25
- loggers.setLevel(logging.INFO)
26
- log_name = './project_M.log'
27
- mkDir(log_name)
28
- logfile = log_name
29
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
30
- time_rotating_file_handler.setLevel(logging.INFO)
31
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
32
- time_rotating_file_handler.setFormatter(formatter)
33
- loggers.addHandler(time_rotating_file_handler)
34
-
35
- """
36
- import tracback
37
- try:
38
-
39
- except Exception as e:
40
-
41
- logger.info(traceback.format_exc())
42
- """
43
-
44
- '''
45
- ##定義一個streamHandler
46
- # print_handler = logging.StreamHandler()
47
- # print_handler.setFormatter(formatter)
48
- # loggers.addHandler(print_handler)
49
- '''
50
-
51
-
52
- """
53
- from main import makeAll,options
54
- from multiprocessing import Pool
55
- import datetime
56
- from config import logger,output
57
- import time
58
- import pandas as pd
59
- import os
60
- from itertools import product
61
- import threading
62
-
63
- def excuteCommand(cmd):
64
- print(cmd)
65
- os.system(cmd)
66
-
67
- def gpuPro(makeListUTC, isPhase, isDebug, gpu, isOverwrite):
68
- productList = product(makeListUTC, [isPhase], [isDebug], [gpu], [isOverwrite])
69
-
70
- with Pool(4) as p:
71
- p.map(makeAll, productList)
72
-
73
- if __name__ == '__main__':
74
- cfg = options()
75
- isPhase = cfg.isPhase
76
- isDebug = cfg.isDebug
77
- sepSec = cfg.sepSec
78
- gpu = cfg.gpu
79
- pool = cfg.pool
80
- isOverwrite = cfg.isOverwrite
81
- timeList = pd.date_range(cfg.times[0], cfg.times[-1], freq=f"{sepSec}s")
82
- logger.info(f"时间段check {timeList}")
83
- gpuNum = 2
84
- eachGPU = 4
85
-
86
- makeListUTC = []
87
- for UTC in timeList:
88
- UTCStr = UTC.strftime("%Y%m%d%H%M")
89
- outpath = f"{output}/{UTCStr[:4]}/{UTCStr[:8]}/MSP2_WTX_AIW_QPF_L88_CHN_{UTCStr}_00000-00300-00006.nc"
90
- if not os.path.exists(outpath) or isOverwrite:
91
- makeListUTC.append(UTC)
92
- [print(element) for element in makeListUTC]
93
-
94
- phaseCMD = "--isPhase" if isPhase else ""
95
- debugCMD = "--isDebug" if isDebug else ""
96
- OverwriteCMD = "--isOverwrite"
97
- gpuCMD = f"--gpu={gpu}"
98
- # cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {OverwriteCMD} {gpuCMD}",makeListUTC))
99
- cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {gpuCMD}",makeListUTC))
100
-
101
- with Pool(pool) as p:
102
- p.map(excuteCommand, cmdList)
103
- """
104
-
105
-
106
- '''
107
- # @Time : 2023/09/27 下午8:52
108
- # @Author : shanchangxi
109
- # @File : util_log.py
110
- import time
111
- import logging
112
- from logging import handlers
113
-
114
- logger = logging.getLogger()
115
- logger.setLevel(logging.INFO)
116
- log_name = 'project_tim_tor.log'
117
- logfile = log_name
118
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
119
- time_rotating_file_handler.setLevel(logging.INFO)
120
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
121
- time_rotating_file_handler.setFormatter(formatter)
122
- logger.addHandler(time_rotating_file_handler)
123
- print_handler = logging.StreamHandler()
124
- print_handler.setFormatter(formatter)
125
- logger.addHandler(print_handler)
126
-
127
- '''
128
-
129
- '''
130
- ###解决方法 pip install torch==2.4.0 torchvision torchaudio三个同时安装 python 3.12 解决cuda启动不了的问题
131
-
132
- Res网络
133
- '''
shancx/N/__init__.py DELETED
@@ -1,131 +0,0 @@
1
- #!/usr/bin/python
2
- # -*- coding: utf-8 -*-
3
- import os
4
- def start():
5
- print("import successful")
6
- # constants
7
-
8
-
9
- __author__ = 'shancx'
10
-
11
- __author_email__ = 'shancx@126.com'
12
-
13
-
14
-
15
- # @Time : 2023/09/27 下午8:52
16
- # @Author : shanchangxi
17
- # @File : util_log.py
18
- import time
19
- import logging
20
- from logging import handlers
21
-
22
- def mkDir(path):
23
- if "." in path:
24
- os.makedirs(os.path.dirname(path),exist_ok=True)
25
- else:
26
- os.makedirs(path, exist_ok=True)
27
-
28
- loggers = logging.getLogger()
29
- loggers.setLevel(logging.INFO)
30
- log_name = '/home/scx/logs/project_N.log'
31
- mkDir(log_name)
32
- logfile = log_name
33
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
34
- time_rotating_file_handler.setLevel(logging.INFO)
35
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
36
- time_rotating_file_handler.setFormatter(formatter)
37
- loggers.addHandler(time_rotating_file_handler)
38
-
39
-
40
- """
41
- import tracback
42
- try:
43
-
44
- except Exception as e:
45
-
46
- logger.info(traceback.format_exc())
47
- """
48
-
49
-
50
- """
51
- from main import makeAll,options
52
- from multiprocessing import Pool
53
- import datetime
54
- from config import logger,output
55
- import time
56
- import pandas as pd
57
- import os
58
- from itertools import product
59
- import threading
60
-
61
- def excuteCommand(cmd):
62
- print(cmd)
63
- os.system(cmd)
64
-
65
- def gpuPro(makeListUTC, isPhase, isDebug, gpu, isOverwrite):
66
- productList = product(makeListUTC, [isPhase], [isDebug], [gpu], [isOverwrite])
67
-
68
- with Pool(4) as p:
69
- p.map(makeAll, productList)
70
-
71
- if __name__ == '__main__':
72
- cfg = options()
73
- isPhase = cfg.isPhase
74
- isDebug = cfg.isDebug
75
- sepSec = cfg.sepSec
76
- gpu = cfg.gpu
77
- pool = cfg.pool
78
- isOverwrite = cfg.isOverwrite
79
- timeList = pd.date_range(cfg.times[0], cfg.times[-1], freq=f"{sepSec}s")
80
- logger.info(f"时间段check {timeList}")
81
- gpuNum = 2
82
- eachGPU = 4
83
-
84
- makeListUTC = []
85
- for UTC in timeList:
86
- UTCStr = UTC.strftime("%Y%m%d%H%M")
87
- outpath = f"{output}/{UTCStr[:4]}/{UTCStr[:8]}/MSP2_WTX_AIW_QPF_L88_CHN_{UTCStr}_00000-00300-00006.nc"
88
- if not os.path.exists(outpath) or isOverwrite:
89
- makeListUTC.append(UTC)
90
- [print(element) for element in makeListUTC]
91
-
92
- phaseCMD = "--isPhase" if isPhase else ""
93
- debugCMD = "--isDebug" if isDebug else ""
94
- OverwriteCMD = "--isOverwrite"
95
- gpuCMD = f"--gpu={gpu}"
96
- # cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {OverwriteCMD} {gpuCMD}",makeListUTC))
97
- cmdList = list(map(lambda x:f"python main.py --times={x.strftime('%Y%m%d%H%M')} {phaseCMD} {debugCMD} {gpuCMD}",makeListUTC))
98
-
99
- with Pool(pool) as p:
100
- p.map(excuteCommand, cmdList)
101
- """
102
-
103
-
104
- '''
105
- # @Time : 2023/09/27 下午8:52
106
- # @Author : shanchangxi
107
- # @File : util_log.py
108
- import time
109
- import logging
110
- from logging import handlers
111
-
112
- logger = logging.getLogger()
113
- logger.setLevel(logging.INFO)
114
- log_name = 'project_tim_tor.log'
115
- logfile = log_name
116
- time_rotating_file_handler = handlers.TimedRotatingFileHandler(filename=logfile, when='D', encoding='utf-8')
117
- time_rotating_file_handler.setLevel(logging.INFO)
118
- formatter = logging.Formatter("%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s")
119
- time_rotating_file_handler.setFormatter(formatter)
120
- logger.addHandler(time_rotating_file_handler)
121
- print_handler = logging.StreamHandler()
122
- print_handler.setFormatter(formatter)
123
- logger.addHandler(print_handler)
124
-
125
- '''
126
-
127
- '''
128
- ###解决方法 pip install torch==2.4.0 torchvision torchaudio三个同时安装 python 3.12 解决cuda启动不了的问题
129
-
130
- Res网络
131
- '''
@@ -1,206 +0,0 @@
1
- import matplotlib.pyplot as plt
2
- import datetime
3
- from hjnwtx.colormap import cmp_hjnwtx
4
- import time
5
- import os
6
- import time
7
- import shutil
8
-
9
- from dateutil.relativedelta import relativedelta
10
- import glob
11
- from config import logger
12
- import argparse
13
- import pandas as pd
14
- import re
15
- import netCDF4 as nc
16
- from multiprocessing import Pool
17
- from itertools import product
18
- import numpy as np
19
- import copy
20
- import traceback
21
-
22
-
23
- from multiprocessing import Process, Queue
24
-
25
- import io
26
- import cartopy.crs as ccrs
27
- import cartopy.feature as cfeature
28
- import cartopy.io.shapereader as shpreader
29
- from hjnwtx.mkNCHJN import mkDir
30
-
31
- shape_ = (1, 5701, 8501)
32
- def add_china_map(ax):
33
- # 在地图上添加地形特征
34
- ax.add_feature(cfeature.COASTLINE, edgecolor='gray')
35
- ax.add_feature(cfeature.BORDERS, linestyle=':', edgecolor='gray')
36
- ax.add_feature(cfeature.LAKES, alpha=0.8)
37
- # 添加省份轮廓
38
- provinces = shpreader.natural_earth(resolution='10m', category='cultural', name='admin_1_states_provinces')
39
- provinces_features = shpreader.Reader(provinces).geometries()
40
- ax.add_geometries(provinces_features, ccrs.PlateCarree(), facecolor='none', edgecolor='gray', linestyle=':', linewidth=0.5, alpha=0.8)
41
-
42
- def draw_subplot(args):
43
- env = [-132.0, -47.0, 0, 57.0]
44
- index, tp, vmax, vmin, cmap,time_index, name = args
45
- # print(time_index)
46
- # logger.info(f"Processing index: {index}")
47
- # logger.info(time_index)
48
- fig, ax = plt.subplots(figsize=(10, 10))
49
-
50
- # 创建绘图和设置坐标系
51
- # fig = plt.figure(figsize=(10, 8))
52
- ax = fig.add_subplot(1, 1, 1, projection=ccrs.PlateCarree())
53
-
54
- # 设置图像显示的范围
55
- ax.set_extent(env, ccrs.PlateCarree()) # 根据需要调整
56
-
57
- # 添加中国地图的边界和特征,包括省份轮廓
58
- add_china_map(ax)
59
- tp = np.where(tp >= 0, tp, np.nan)
60
- ax.imshow(tp, vmin=0, vmax=100, cmap=cmp_hjnwtx["radar_nmc"], transform=ccrs.PlateCarree(), extent=env, alpha=1)
61
-
62
-
63
- # ax.imshow(tp, vmax=vmax, vmin=vmin, cmap=cmap)
64
- ax.axis('off')
65
- # ax.text(0.5, 0.5, str(index), transform=ax.transAxes, color='white', fontsize=20, ha='center', va='center')
66
- ax.text(0.95, 0.95, f'{time_index}', transform=ax.transAxes, color='white', fontsize=20, ha='right', va='bottom')
67
- ax.text(0.925, 0.925, f'{str(index)}', transform=ax.transAxes, color='white', fontsize=20, ha='right', va='top')
68
- fig.tight_layout()
69
-
70
- buf = io.BytesIO()
71
- plt.savefig(buf, format='png')
72
- buf.seek(0)
73
- plt.close(fig)
74
- return (index, buf) #返回两个数为元组
75
-
76
- # 修改读取部分
77
- def drawpic(tp, Count,timeList, name="temp"):
78
- vmax = 70
79
- vmin = 0
80
- cmap = cmp_hjnwtx["radar_nmc"]
81
- args_list = [ (index, tp[index, :, :], vmax, vmin, cmap,timeList[index], name) for index in range(2 * Count)]
82
-
83
- with Pool(31) as pool:
84
- results = pool.map(draw_subplot, args_list)
85
-
86
- buffers = [result[1] for result in results]
87
- combined_fig, axs = plt.subplots(2, Count, figsize=(10 * Count, 20))
88
- for i in range(2):
89
- for j in range(Count):
90
- index = i * Count + j
91
- ax = axs[i, j]
92
- ax.imshow(plt.imread(buffers[index]), cmap=cmap)
93
- ax.axis('off')
94
-
95
- now_str = datetime.datetime.now().strftime("%Y%m%d%H%M")
96
- plt.tight_layout()
97
- outpath = f"./{timeList[0].strftime('%Y%m%d%H%M')}_CR/{name}_{now_str}.png"
98
- mkDir(outpath)
99
- plt.savefig(outpath)
100
- plt.close(combined_fig)
101
-
102
-
103
-
104
- BASE_TARGET_PATH = "/mnt/wtx_weather_forecast/WTX_DATA/RADA/MQPF_GLB" #/mnt/wtx_weather_forecast/WTX_DATA/RADA/MQPF_PMSC
105
- # BASE_TARGET_PATH = "/mnt/wtx_weather_forecast/WTX_DATA/RADA/MQPF_PMSC" #/mnt/wtx_weather_forecast/WTX_DATA/RADA/MQPF_PMSC
106
-
107
- def get_mqpf_paths(UCTstr):
108
- year = UCTstr[:4]
109
- date = UCTstr[:8]
110
- mqpfPath_pattern = os.path.join(BASE_TARGET_PATH,year, date,f"MSP2_WTX_AIW_REF_L88_NA_{UCTstr}_00000-00300-00006.nc") #MSP2_WTX_AIW_QPF_L88_NA_202410222254_00000-00300-00006.nc
111
- # mqpfPath_pattern = os.path.join(BASE_TARGET_PATH,year, date,f"MSP2_WTX_AIW_MQPF_L88_CHN_{UCTstr}_00000-00300-00006.nc")
112
-
113
- return mqpfPath_pattern
114
-
115
- def map_data(conf):
116
- CST = conf[0]
117
- UCT = CST + relativedelta(hours=-8)
118
- UCTstr = UCT.strftime("%Y%m%d%H%M")
119
- mqpfPath_pattern = get_mqpf_paths(UCTstr)
120
- mqpfPath_list = glob.glob(mqpfPath_pattern)
121
- if len(mqpfPath_list) == 0:
122
- data_loss = re.findall(r"(2024\d{8}?)",mqpfPath_pattern)
123
- print("data_loss",data_loss)
124
- print("data_loss_path",mqpfPath_pattern)
125
- tj_list.append(data_loss[0])
126
- return np.full(shape_, np.nan)
127
- else:
128
- try:
129
- with nc.Dataset(mqpfPath_list[0]) as dataNC:
130
- mqpf = dataNC.variables["CR"][:]
131
- mqpf = mqpf[:1]
132
- if mqpf.shape != shape_:
133
- logger.info(mqpf.shape )
134
- logger.info(mqpfPath_list[0])
135
- print("mqpf shape error",mqpf.shape)
136
- mqpf = mqpf[:,:-1, :-1]
137
- if mqpf.shape != shape_:
138
- return np.full(shape_, np.nan)
139
- else:
140
- return mqpf
141
- print("mqpf",UCTstr,mqpf.shape)
142
- tj_list1.append(mqpfPath_pattern)
143
- return mqpf
144
- except Exception as e:
145
- print(traceback.format_exc())
146
- return np.full(shape_, np.nan)
147
-
148
- def options():
149
- parser = argparse.ArgumentParser(description='plot radar')
150
- # parser.add_argument('--times', type=str, default='202406290000,202406300000')
151
- parser.add_argument('--times', type=str, default='202407310000,202408010000')
152
- parser.add_argument('--pac', type=str, default='100000')
153
- # parser.add_argument('--combine', action='store_true', default=False)
154
- parser.add_argument('--combine',action='store_true',default=False)
155
- parser.add_argument('--isDebug',action='store_true',default=False)
156
- parser.add_argument('--isDraw',action='store_true',default=False)
157
- parser.add_argument('--freq', type=str, default="1h")
158
- parser.add_argument('--tag',type=str, default=datetime.datetime.now().strftime("%Y%m%d%H%M"))
159
- config= parser.parse_args()
160
- print(config)
161
- config.times = config.times.split(",")
162
- config.pac = config.pac.split(",")
163
- if len(config.times) == 1:
164
- config.times = [config.times[0], config.times[0]]
165
- config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
166
- datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
167
- return config
168
- if __name__ == '__main__':
169
- cfg = options()
170
- sCST = cfg.times[0]
171
- eCST = cfg.times[-1]
172
- sCSTstr = sCST.strftime("%Y%m%d")
173
- tj_list = []
174
- tj_list1= []
175
- start = datetime.datetime.now()
176
- logger.info(f"{sCST}")
177
- logger.info(f"{eCST}")
178
- logger.info(f"PNG")
179
- timeList = pd.date_range(sCST,eCST,freq="360s",inclusive="left")
180
- # for CST in timeList:
181
- productList = product(timeList)
182
- with Pool(31) as p:
183
- Data = p.map(map_data,productList)
184
- end = datetime.datetime.now()
185
- print(start-end)
186
- Data_con = np.concatenate(Data,axis=0)
187
- loss_len = 240 - Data_con.shape[0]
188
- sCSTstr = sCST.strftime("%Y%m%d")
189
- eCSTstr = eCST.strftime("%Y%m%d")
190
- # Data_con1 = Data_con.filled()
191
- # np.save(f"data_{sCSTstr}_{eCSTstr}.npy",Data_con1)
192
- Data_con_120 = copy.copy(Data_con)
193
- drawpic(Data_con_120[:int(len(Data_con_120)/2)], int(len(Data_con_120)/4),timeList[:int(len(Data_con_120)/2)],name=f"temp120_{sCSTstr}_loss_{loss_len}_")
194
- print("done 120")
195
- end1 = datetime.datetime.now()
196
- print(end1-end)
197
- Data_con_240 = copy.copy(Data_con)
198
- print(Data_con_240[120:].shape)
199
- drawpic(Data_con_120[int(len(Data_con_120)/2):], int(len(Data_con_120)/4),timeList[int(len(Data_con_120)/2):],name=f"temp120_240_{sCSTstr}_{loss_len}_")
200
- print(datetime.datetime.now()-end1)
201
- print("done 120-240")
202
- logger.info("success")
203
-
204
-
205
- # "/mnt/wtx_weather_forecast/WTX_DATA/RADA/MQPF/2024/20240704/MSP2_WTX_AIW_REF_L88_CHN_202407040324_00000-00300-00006.nc"
206
-