shancx 1.9.33.220__py3-none-any.whl → 1.9.33.222__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- shancx/NN/chainMul.py +175 -0
- {shancx-1.9.33.220.dist-info → shancx-1.9.33.222.dist-info}/METADATA +1 -1
- {shancx-1.9.33.220.dist-info → shancx-1.9.33.222.dist-info}/RECORD +5 -4
- {shancx-1.9.33.220.dist-info → shancx-1.9.33.222.dist-info}/WHEEL +0 -0
- {shancx-1.9.33.220.dist-info → shancx-1.9.33.222.dist-info}/top_level.txt +0 -0
shancx/NN/chainMul.py
ADDED
|
@@ -0,0 +1,175 @@
|
|
|
1
|
+
import subprocess
|
|
2
|
+
import time
|
|
3
|
+
import logging
|
|
4
|
+
import os
|
|
5
|
+
import argparse
|
|
6
|
+
import datetime
|
|
7
|
+
import traceback
|
|
8
|
+
from dateutil.relativedelta import relativedelta
|
|
9
|
+
import pandas as pd
|
|
10
|
+
from shancx.NN import Mul_TH
|
|
11
|
+
from shancx.NN import _loggersPlus
|
|
12
|
+
from shancx.NN import cleanupLogs
|
|
13
|
+
from shancx.sendM import sendMES
|
|
14
|
+
BASE_DIR = '/mnt/wtx_weather_forecast/scx/sever7/product_log_scx/code06_re2_cp1/exam/mk_trainsatRadarMSG1'
|
|
15
|
+
PYTHON_ENV = '/home/scx/miniconda3/envs/mqpf/bin/python'
|
|
16
|
+
DEFAULT_GPU = '0'
|
|
17
|
+
DEFAULT_TIMEOUT = 2400
|
|
18
|
+
SCRIPT_DELAY = 2
|
|
19
|
+
# minutedict = {"20":8,"30":6,"40":4,"50":2,"0":0}
|
|
20
|
+
minutedict = {"0": 0 }
|
|
21
|
+
class PipelineExecutor:
|
|
22
|
+
def __init__(self, UTC, sat_cd, gpu=DEFAULT_GPU):
|
|
23
|
+
self.UTCstr = UTC.strftime("%Y%m%d%H%M")
|
|
24
|
+
self.sat_cd = sat_cd
|
|
25
|
+
self.gpu = gpu
|
|
26
|
+
self.produce_time= self.producetime(UTC)
|
|
27
|
+
self.scripts = self._initialize_scripts()
|
|
28
|
+
def producetime(self,UTC):
|
|
29
|
+
_minute = UTC.minute
|
|
30
|
+
_minute = minutedict.get(str(_minute),None)
|
|
31
|
+
if _minute is not None :
|
|
32
|
+
start_time = UTC - relativedelta(minutes=_minute) # "20":8 indicates that the step 2 command is executed at the 12-minute mark, which is between 20 and 10 minutes.
|
|
33
|
+
return start_time
|
|
34
|
+
else:
|
|
35
|
+
return None
|
|
36
|
+
def _initialize_scripts(self):
|
|
37
|
+
#step 1 command
|
|
38
|
+
scripts = [
|
|
39
|
+
{
|
|
40
|
+
'name': 'MSGsat',
|
|
41
|
+
'cmd': f"cd {os.path.join(BASE_DIR, 'makeMSGnc')} && {PYTHON_ENV} MSGsat.py --times {self.UTCstr[:12]}",
|
|
42
|
+
'timeout': DEFAULT_TIMEOUT
|
|
43
|
+
}
|
|
44
|
+
]
|
|
45
|
+
#step 2 command
|
|
46
|
+
if self.produce_time is not None:
|
|
47
|
+
times_str = self.produce_time.strftime("%Y%m%d%H%M")[:12]
|
|
48
|
+
scripts.extend([
|
|
49
|
+
# {
|
|
50
|
+
# 'name': 'mainSAT',
|
|
51
|
+
# 'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9SEAS --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9SEAS'",
|
|
52
|
+
# 'timeout': DEFAULT_TIMEOUT
|
|
53
|
+
# },
|
|
54
|
+
# {
|
|
55
|
+
# 'name': 'mainSAT',
|
|
56
|
+
# 'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9CHNNEAS --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9OC'",
|
|
57
|
+
# 'timeout': DEFAULT_TIMEOUT
|
|
58
|
+
# },
|
|
59
|
+
# {
|
|
60
|
+
# 'name': 'mainSAT',
|
|
61
|
+
# 'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9OC --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9CHNNEAS'",
|
|
62
|
+
# 'timeout': DEFAULT_TIMEOUT
|
|
63
|
+
# },
|
|
64
|
+
# # [
|
|
65
|
+
# # # {
|
|
66
|
+
# # # 'name': 'satPzlSATREF',
|
|
67
|
+
# # # 'cmd': f"cd {os.path.join(BASE_DIR, 'SawPuz')} && {PYTHON_ENV} satPzlSAT.py --times {times_str[:12]} --satcd '{self.sat_cd}' --mqpf_cd 'REF'",
|
|
68
|
+
# # # 'timeout': DEFAULT_TIMEOUT
|
|
69
|
+
# # # },
|
|
70
|
+
# # # {
|
|
71
|
+
# # # 'name': 'satPzlSATQPF',
|
|
72
|
+
# # # 'cmd': f"cd {os.path.join(BASE_DIR, 'SawPuz')} && {PYTHON_ENV} satPzlSAT.py --times {times_str[:12]} --satcd '{self.sat_cd}' --mqpf_cd 'QPF' --isUpload ",
|
|
73
|
+
# # # 'timeout': DEFAULT_TIMEOUT
|
|
74
|
+
# # # }
|
|
75
|
+
# # ]
|
|
76
|
+
])
|
|
77
|
+
else:
|
|
78
|
+
logger.warning("produce_time is None, skipping mainSAT, satPzlSATREF and satPzlSATQPF command execution")
|
|
79
|
+
return scripts
|
|
80
|
+
def execute_script(self, conf):
|
|
81
|
+
script_info = conf[0]
|
|
82
|
+
script_name = script_info['name']
|
|
83
|
+
timeout = script_info['timeout']
|
|
84
|
+
logger.info(f"Starting to execute script: {script_name}")
|
|
85
|
+
try:
|
|
86
|
+
result = subprocess.run(
|
|
87
|
+
script_info['cmd'],
|
|
88
|
+
shell=True,
|
|
89
|
+
timeout=timeout,
|
|
90
|
+
check=False,
|
|
91
|
+
stdout=subprocess.PIPE,
|
|
92
|
+
stderr=subprocess.PIPE,
|
|
93
|
+
text=True
|
|
94
|
+
)
|
|
95
|
+
if result.returncode != 0:
|
|
96
|
+
logger.error(f"Script {script_name} execution failed, return code: {result.returncode}")
|
|
97
|
+
if result.stdout.strip():
|
|
98
|
+
logger.error(f"Output before failure:\n{result.stdout}")
|
|
99
|
+
if result.stderr.strip():
|
|
100
|
+
logger.error(f"Error output:\n{result.stderr}")
|
|
101
|
+
logger.error(f"Traceback:\n{traceback.format_exc()}")
|
|
102
|
+
return False
|
|
103
|
+
# Success case
|
|
104
|
+
if result.stdout.strip():
|
|
105
|
+
logger.debug(f"[{script_name}] Output:\n{result.stdout}")
|
|
106
|
+
if result.stderr.strip():
|
|
107
|
+
logger.warning(f"[{script_name}] Warning output:\n{result.stderr}")
|
|
108
|
+
logger.info(f"Script {script_name} executed successfully")
|
|
109
|
+
return True
|
|
110
|
+
except Exception as e:
|
|
111
|
+
logger.error(f"Script {script_name} execution exception: {str(e)}")
|
|
112
|
+
logger.error(f"Traceback:\n{traceback.format_exc()}")
|
|
113
|
+
sendMES(f"{traceback.format_exc()} ",NN="MT1")
|
|
114
|
+
return False
|
|
115
|
+
def run_pipeline(self):
|
|
116
|
+
"""Execute complete script pipeline"""
|
|
117
|
+
logger.info(f"Starting to process time slot: {self.UTCstr}, satellite: {self.sat_cd}")
|
|
118
|
+
for i, script in enumerate(self.scripts, 1):
|
|
119
|
+
if isinstance(script, list):
|
|
120
|
+
logger.info(f"Executing step {i}/{len(self.scripts)}: satPzlSAT")
|
|
121
|
+
Mul_TH(self.execute_script,[script])
|
|
122
|
+
else:
|
|
123
|
+
logger.info(f"Executing step {i}/{len(self.scripts)}: {script['name']}")
|
|
124
|
+
if not self.execute_script((script,)):
|
|
125
|
+
logger.error(f"Pipeline interrupted at {script['name']}, processing failed")
|
|
126
|
+
return False
|
|
127
|
+
if i < len(self.scripts):
|
|
128
|
+
time.sleep(SCRIPT_DELAY)
|
|
129
|
+
logger.info(f"Time slot {self.UTCstr} processing completed")
|
|
130
|
+
return True
|
|
131
|
+
def main(conf):
|
|
132
|
+
utc,sat_cd,gpu = conf[0],conf[1],conf[2]
|
|
133
|
+
executor = PipelineExecutor(utc, sat_cd, gpu)
|
|
134
|
+
return executor.run_pipeline()
|
|
135
|
+
def options():
|
|
136
|
+
parser = argparse.ArgumentParser(description='Pipeline Cascade time command execution')
|
|
137
|
+
parser.add_argument('--times', type=str, default='202412010000,202508010000')
|
|
138
|
+
parser.add_argument('--sat_cd', type=str, default='MSG')
|
|
139
|
+
parser.add_argument('--gpu', type=str, default='0')
|
|
140
|
+
config= parser.parse_args()
|
|
141
|
+
print(config)
|
|
142
|
+
config.times = config.times.split(",")
|
|
143
|
+
if len(config.times) == 1:
|
|
144
|
+
config.times = [config.times[0], config.times[0]]
|
|
145
|
+
config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
|
|
146
|
+
datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
|
|
147
|
+
return config
|
|
148
|
+
if __name__ == "__main__":
|
|
149
|
+
cfg = options()
|
|
150
|
+
sUTC = cfg.times[0]
|
|
151
|
+
eUTC = cfg.times[-1]
|
|
152
|
+
sat_cd = cfg.sat_cd
|
|
153
|
+
gpu =cfg.gpu
|
|
154
|
+
sUTCstr = sUTC.strftime("%Y%m%d%H%M")
|
|
155
|
+
dir_ = f"./logs/{sat_cd}"
|
|
156
|
+
logger = _loggersPlus(root = dir_ , phase=f"{sUTCstr}_Pipeline")
|
|
157
|
+
from shancx.NN import cleanupLogs
|
|
158
|
+
stats = cleanupLogs(dir_,30, '*.log', False, False)
|
|
159
|
+
logging.info(f"clean={stats['total_dirs']}, cleanfile={stats['deleted_files']}, error={len(stats['errors'])}")
|
|
160
|
+
|
|
161
|
+
timeList = pd.date_range(sUTC, eUTC, freq='30min')
|
|
162
|
+
# filtered_times = [t for t in timeList if t.hour % 4 == 0 and t.minute == 0] #t.mintues
|
|
163
|
+
# result_times = []
|
|
164
|
+
# for t in filtered_times:
|
|
165
|
+
# result_times.extend(pd.date_range(t - pd.Timedelta(minutes=30), t, freq='10T'))
|
|
166
|
+
# for UTC in result_times:
|
|
167
|
+
# main((UTC,sat_cd,gpu))
|
|
168
|
+
Mul_TH(main,[timeList,[sat_cd],[gpu]],2)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
|
|
@@ -66,6 +66,7 @@ shancx/H9/geometry.py,sha256=ZoCfgP07ANqDxyLf8uFN016DPl7QqW9pAV3sr2ekCng,95893
|
|
|
66
66
|
shancx/Hug/__init__.py,sha256=NvcqXM0CSwXYtTr5yj4zfkag_7PZcI5tCsIHjuLNXTQ,3040
|
|
67
67
|
shancx/Mos/__init__.py,sha256=IDlCLNN6a_mguYmRcAPcevHSDaud5ZhvYNSuASqztcE,1563
|
|
68
68
|
shancx/NN/__init__.py,sha256=ENgZxIzUJzkuxArkcH-1JlxQA08Urq2ppFkGCbIURZk,9219
|
|
69
|
+
shancx/NN/chainMul.py,sha256=p7q_jrgFQkkfAYr_ZJFirwZOYoyn31QOVYERwhBGH4k,8649
|
|
69
70
|
shancx/Plot/GlobMap.py,sha256=WRvZOf5bo8IfECk1-sN4g6K0CrcJhSoEJAiLvZAWDwg,13021
|
|
70
71
|
shancx/Plot/Gray2RGB.py,sha256=yDXu1xWVi7OtrS8_ExxK_ev1988kj1ZxoNCWf-kl4Zc,2734
|
|
71
72
|
shancx/Plot/__init__.py,sha256=_fmDFZfh9HzDPBPKCMPrDT2DsblLVfrO-uFvUg76a74,19194
|
|
@@ -86,7 +87,7 @@ shancx/Train/multiGpu.py,sha256=D_oZeiSc7VWktpnVDwrFOC1CYZSt9rxOKY5lngE5vFg,820
|
|
|
86
87
|
shancx/Train/prepare.py,sha256=vL_8UOA66oZCBIwCICtihsGibivtNgaVJGulJxfNdn8,6793
|
|
87
88
|
shancx/Train/renet50.py,sha256=wEhYk1X96WE5zuqHqVxWLJa-A5jDNkz4z6edORNufnA,6428
|
|
88
89
|
shancx/tensBoard/__init__.py,sha256=ga2C5YyJITvvQA1ocpxna_KNFnNRJVwkTjLoIglLZUQ,993
|
|
89
|
-
shancx-1.9.33.
|
|
90
|
-
shancx-1.9.33.
|
|
91
|
-
shancx-1.9.33.
|
|
92
|
-
shancx-1.9.33.
|
|
90
|
+
shancx-1.9.33.222.dist-info/METADATA,sha256=vnbm-h0oqRzFukBJQK-CJTJxGCY7DbC1mOmEZ9LO0-I,850
|
|
91
|
+
shancx-1.9.33.222.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
|
|
92
|
+
shancx-1.9.33.222.dist-info/top_level.txt,sha256=akfCS1vKWz3pNmEN_yN9ZiGp-60IQY5ET38mRx_i_-4,7
|
|
93
|
+
shancx-1.9.33.222.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|