shancx 1.9.33.220__py3-none-any.whl → 1.9.33.226__py3-none-any.whl

shancx/Hug/__init__.py

@@ -44,9 +44,6 @@ model_dir = snapshot_download(
 print(f"模型已下载到: {model_dir}")
 
 
-
-
-
 from google.colab import drive
 import requests
 import os

shancx/NN/chainMul.py

@@ -0,0 +1,175 @@
+import subprocess
+import time
+import logging
+import os
+import argparse
+import datetime
+import traceback
+from dateutil.relativedelta import relativedelta
+import pandas as pd 
+from shancx.NN import Mul_TH
+from shancx.NN import _loggersPlus
+from shancx.NN import cleanupLogs
+from shancx.sendM import sendMES
+BASE_DIR = '/mnt/wtx_weather_forecast/scx/sever7/product_log_scx/code06_re2_cp1/exam/mk_trainsatRadarMSG1'
+PYTHON_ENV = '/home/scx/miniconda3/envs/mqpf/bin/python'
+DEFAULT_GPU = '0'
+DEFAULT_TIMEOUT = 2400
+SCRIPT_DELAY = 2
+# minutedict = {"20":8,"30":6,"40":4,"50":2,"0":0}   
+minutedict = {"0": 0 }   
+class PipelineExecutor:  
+    def __init__(self, UTC, sat_cd, gpu=DEFAULT_GPU):
+        self.UTCstr = UTC.strftime("%Y%m%d%H%M")
+        self.sat_cd = sat_cd
+        self.gpu = gpu
+        self.produce_time= self.producetime(UTC)
+        self.scripts = self._initialize_scripts()    
+    def producetime(self,UTC):   
+        _minute = UTC.minute
+        _minute = minutedict.get(str(_minute),None)
+        if _minute is not None :
+            start_time = UTC - relativedelta(minutes=_minute)  # "20":8 indicates that the step 2 command is executed at the 12-minute mark, which is between 20 and 10 minutes.
+            return start_time
+        else:
+            return None
+    def _initialize_scripts(self):
+        #step 1 command 
+        scripts = [            
+            {
+                'name': 'MSGsat',
+                'cmd': f"cd {os.path.join(BASE_DIR, 'makeMSGnc')} && {PYTHON_ENV} MSGsat.py --times {self.UTCstr[:12]}",
+                'timeout': DEFAULT_TIMEOUT
+            }
+        ]
+        #step 2 command 
+        if self.produce_time is not None:
+            times_str = self.produce_time.strftime("%Y%m%d%H%M")[:12]
+            scripts.extend([
+                            #   {
+                            #       'name': 'mainSAT',
+                            #       'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9SEAS --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9SEAS'",
+                            #       'timeout': DEFAULT_TIMEOUT
+                            #   },
+                            #   {
+                            #       'name': 'mainSAT',
+                            #       'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9CHNNEAS --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9OC'",
+                            #       'timeout': DEFAULT_TIMEOUT
+                            #   },
+                            #   {
+                            #       'name': 'mainSAT',
+                            #       'cmd': f"cd {os.path.join('/mnt/wtx_weather_forecast/scx', 'mqpf_GEOSH9SEAStestCal')} && {PYTHON_ENV} mainSAT.py --times {times_str[:12]} --satcd H9OC --sepSec 360 --gpu '{self.gpu}' --isOverwrite --classer 'WT_RTH9CHNNEAS'",
+                            #       'timeout': DEFAULT_TIMEOUT
+                            #   },
+                            # #   [
+                            # #     #  {
+                            # #     #   'name': 'satPzlSATREF',
+                            # #     #   'cmd': f"cd {os.path.join(BASE_DIR, 'SawPuz')} && {PYTHON_ENV} satPzlSAT.py --times {times_str[:12]} --satcd '{self.sat_cd}' --mqpf_cd 'REF'",
+                            # #     #   'timeout': DEFAULT_TIMEOUT
+                            # #     #  },
+                            # #     #  {
+                            # #     #   'name': 'satPzlSATQPF',
+                            # #     #   'cmd': f"cd {os.path.join(BASE_DIR, 'SawPuz')} && {PYTHON_ENV} satPzlSAT.py --times {times_str[:12]} --satcd '{self.sat_cd}' --mqpf_cd 'QPF' --isUpload ",
+                            # #     #   'timeout': DEFAULT_TIMEOUT  
+                            # #     #  }    
+                            # #   ] 
+                           ])
+        else:
+            logger.warning("produce_time is None, skipping mainSAT, satPzlSATREF and satPzlSATQPF command execution")        
+        return scripts
+    def execute_script(self, conf): 
+        script_info = conf[0] 
+        script_name = script_info['name']
+        timeout = script_info['timeout']      
+        logger.info(f"Starting to execute script: {script_name}")       
+        try:
+            result = subprocess.run(
+                                     script_info['cmd'],
+                                     shell=True,
+                                     timeout=timeout,
+                                     check=False,  
+                                     stdout=subprocess.PIPE,
+                                     stderr=subprocess.PIPE,
+                                     text=True
+                                   )            
+            if result.returncode != 0:
+                logger.error(f"Script {script_name} execution failed, return code: {result.returncode}")
+                if result.stdout.strip():
+                    logger.error(f"Output before failure:\n{result.stdout}")
+                if result.stderr.strip():
+                    logger.error(f"Error output:\n{result.stderr}")
+                logger.error(f"Traceback:\n{traceback.format_exc()}")
+                return False            
+            # Success case
+            if result.stdout.strip():
+                logger.debug(f"[{script_name}] Output:\n{result.stdout}")
+            if result.stderr.strip():
+                logger.warning(f"[{script_name}] Warning output:\n{result.stderr}")               
+            logger.info(f"Script {script_name} executed successfully")
+            return True            
+        except Exception as e:
+            logger.error(f"Script {script_name} execution exception: {str(e)}")
+            logger.error(f"Traceback:\n{traceback.format_exc()}")
+            sendMES(f"{traceback.format_exc()} ",NN="MT1")
+            return False    
+    def run_pipeline(self):
+        """Execute complete script pipeline"""
+        logger.info(f"Starting to process time slot: {self.UTCstr}, satellite: {self.sat_cd}")        
+        for i, script in enumerate(self.scripts, 1): 
+            if isinstance(script, list):  
+                logger.info(f"Executing step {i}/{len(self.scripts)}: satPzlSAT") 
+                Mul_TH(self.execute_script,[script])
+            else:   
+                logger.info(f"Executing step {i}/{len(self.scripts)}: {script['name']}")     
+                if not self.execute_script((script,)):
+                    logger.error(f"Pipeline interrupted at {script['name']}, processing failed")
+                    return False
+            if i < len(self.scripts):
+                time.sleep(SCRIPT_DELAY)        
+        logger.info(f"Time slot {self.UTCstr} processing completed")
+        return True
+def main(conf): 
+    utc,sat_cd,gpu = conf[0],conf[1],conf[2]
+    executor = PipelineExecutor(utc, sat_cd, gpu)
+    return executor.run_pipeline()
+def options():
+    parser = argparse.ArgumentParser(description='Pipeline Cascade time command execution')
+    parser.add_argument('--times', type=str, default='202412010000,202508010000') 
+    parser.add_argument('--sat_cd', type=str, default='MSG') 
+    parser.add_argument('--gpu', type=str, default='0')
+    config= parser.parse_args()
+    print(config)
+    config.times = config.times.split(",")
+    if len(config.times) == 1:
+        config.times = [config.times[0], config.times[0]]
+    config.times = [datetime.datetime.strptime(config.times[0], "%Y%m%d%H%M"),
+                    datetime.datetime.strptime(config.times[1], "%Y%m%d%H%M")]
+    return config
+if __name__ == "__main__":
+    cfg = options()
+    sUTC = cfg.times[0]
+    eUTC = cfg.times[-1]
+    sat_cd  = cfg.sat_cd
+    gpu =cfg.gpu
+    sUTCstr = sUTC.strftime("%Y%m%d%H%M")
+    dir_ = f"./logs/{sat_cd}"
+    logger = _loggersPlus(root = dir_ , phase=f"{sUTCstr}_Pipeline")
+    from shancx.NN import cleanupLogs
+    stats = cleanupLogs(dir_,30, '*.log', False, False) 
+    logging.info(f"clean={stats['total_dirs']}, cleanfile={stats['deleted_files']}, error={len(stats['errors'])}") 
+
+    timeList = pd.date_range(sUTC, eUTC, freq='30min')
+    # filtered_times = [t for t in timeList if t.hour  % 4 == 0 and t.minute == 0]  #t.mintues
+    # result_times = []
+    # for t in filtered_times:
+    #     result_times.extend(pd.date_range(t - pd.Timedelta(minutes=30), t, freq='10T')) 
+    # for UTC in result_times:
+    #     main((UTC,sat_cd,gpu))
+    Mul_TH(main,[timeList,[sat_cd],[gpu]],2)
+
+
+
+
+ 
+ 
+ 

shancx/Read.py

@@ -1,9 +1,77 @@
+import xarray as xr
+import numpy as np
+import gzip
+import tempfile
+import os
+def getGrib(filepath):
+    tmp_path = filepath 
+    if filepath.endswith('.gz'):
+        with gzip.open(filepath, 'rb') as f:
+            with tempfile.NamedTemporaryFile(suffix='.grib2', delete=False) as tmp:
+                tmp.write(f.read())
+                tmp_path = tmp.name   
+    try:
+        ds = xr.open_dataset(tmp_path, engine='cfgrib')
+        var_name = list(ds.data_vars)[0]
+        data_var = ds[var_name] 
+        data_values = np.where(data_var.values < -900, np.nan, data_var.values)
+        if 'latitude' in ds.coords and 'longitude' in ds.coords:
+            lats = ds.latitude.values
+            lons = ds.longitude.values
+        elif 'lat' in ds.coords and 'lon' in ds.coords:
+            lats = ds.lat.values
+            lons = ds.lon.values
+        elif hasattr(ds, 'latitude') and hasattr(ds, 'longitude'):
+            lats = ds.latitude.values
+            lons = ds.longitude.values
+        else:
+            for var_name in ds.data_vars:
+                var = ds[var_name]
+                if hasattr(var, 'latitude') and hasattr(var, 'longitude'):
+                    lats = var.latitude.values
+                    lons = var.longitude.values
+                    break
+            else:
+                raise ValueError("未找到经纬度坐标")
+        lat_range = (float(np.nanmin(lats)), float(np.nanmax(lats)))
+        lon_range = (float(np.nanmin(lons)), float(np.nanmax(lons)))        
+        return {
+            'data': data,
+            'lats': lats,
+            'lons': lons 
+        }        
+    finally:
+        os.unlink(tmp_path)
+"""
+input_Path = './Composite_00.50_20250701-010044.grib2.gz'
+result = getGrib(input_Path)
+"""
+
 import pygrib
 import numpy as np
 import pandas as pd
 def readGrib(file_path, target_param=None):
     try:
-        with pygrib.open(file_path) as grbs:
+        with pygrib.open(file_path) as grbs:    
+            file_path= file_path  
+            if filepath.endswith('.gz'):
+                with gzip.open(filepath, 'rb') as f:
+                    with tempfile.NamedTemporaryFile(suffix='.grib2', delete=False) as tmp:
+                        tmp.write(f.read())
+                        file_path = tmp.name       
+            if target_param:
+                try:
+                    grb = grbs.select(shortName=target_param)[0]
+                except:
+                    try:
+                        grb = grbs.select(parameterName=target_param)[0]
+                    except:
+                        raise ValueError(f"未找到参数: {target_param}")
+            else:
+                grb = grbs[1]
+            data = grb.values
+            lats, lons = grb.latlons() 
+            #data2 
             field_info = []
             for grb in grbs:
                 field_info.append({
@@ -16,18 +84,6 @@ def readGrib(file_path, target_param=None):
                                    'units': getattr(grb, 'units', 'N/A'),
                                    'shape': grb.values.shape
                                  })             
-            if target_param:
-                try:
-                    grb = grbs.select(shortName=target_param)[0]
-                except:
-                    try:
-                        grb = grbs.select(parameterName=target_param)[0]
-                    except:
-                        raise ValueError(f"未找到参数: {target_param}")
-            else:
-                grb = grbs[1]
-            data = grb.values
-            lats, lons = grb.latlons()            
             return {
                 'data': data,
                 'lats': lats,
@@ -36,7 +92,8 @@ def readGrib(file_path, target_param=None):
                     'parameterName': grb.parameterName,
                     'level': grb.level,
                     'validDate': grb.validDate,
-                    'units': grb.units
+                    'units': grb.units,
+                    "field_info":field_info                    
                 }
             }            
     except Exception as e:
@@ -50,9 +107,5 @@ if __name__ == "__main__":
         print("经度范围:", np.min(result['lons']), "~", np.max(result['lons']))
         print("纬度范围:", np.min(result['lats']), "~", np.max(result['lats']))
         print("参数单位:", result['metadata']['units'])
-        """
-        latArr = latMat[:,0]
-        lonArr = lonMat[0]
-        """    
+  
 
- 

shancx/sendM.py

@@ -41,14 +41,13 @@ def sendMES(message,key='0f313-17b2-4e3d-84b8-3f9c290fa596',NN = None):
     elif NN=="MSGAF":
         webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=5797fbe9-1c19-4a67-a6fc-83a85cb464a7"      
     elif NN=="GOES18C":
-        webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=4ddac32b-692b-4aa0-9d69-0e655552ff17"   
-        
-
+        webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=4ddac32b-692b-4aa0-9d69-0e655552ff17"  
     elif NN=="GFS":
-        webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=13f39c2f-e191-4100-b1ee-7316ac9c2451"    
-    else:
         webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=13f39c2f-e191-4100-b1ee-7316ac9c2451"
-        
+    elif NN=="WTX":
+        webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=44869d2b-ab46-4bad-9621-4cda03470908"     
+    else:
+        webHookUrl ="https://qyapi.weixin.qq.com/cgi-bin/webhook/send?key=13f39c2f-e191-4100-b1ee-7316ac9c2451"        
     try:
         url=webHookUrl
         headers = {"Content-Type":"application/json"}
@@ -56,4 +55,4 @@ def sendMES(message,key='0f313-17b2-4e3d-84b8-3f9c290fa596',NN = None):
         res = requests.post(url,json=data,headers=headers)
     except Exception as e:
         print(e)
- 
+ #sendMES(f" {sat_cd} {UTC}",NN="WTX") 

{shancx-1.9.33.220.dist-info → shancx-1.9.33.226.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: shancx
-Version: 1.9.33.220
+Version: 1.9.33.226
 Summary: A simple timer decorator
 Home-page: https://gitee.com/shancx
 Author: shancx

{shancx-1.9.33.220.dist-info → shancx-1.9.33.226.dist-info}/RECORD

@@ -5,7 +5,7 @@ shancx/Lib.py,sha256=GUAspllSxk39mvj-F1Q8Ys0EcY_lQfZPRGPE7L3x4SE,977
 shancx/Path1.py,sha256=vX4A5RgdwVyIVVNZRocl18rOu1Z8YMLfDb0B92-0bE8,5334
 shancx/Point.py,sha256=gyIomOVbNoz6SOcAhhTS26NHBeJ0TOwB-ljNsBWN1ZE,1909
 shancx/QC.py,sha256=MFxbPMEwrnk5l0sTFMrx13wPlXJ5WI5TovzaiBoEr0E,9848
-shancx/Read.py,sha256=qaX4Qf9J8vXjam60PWjWdAL-1hvEwcEe_d4bR4MZ2OU,2465
+shancx/Read.py,sha256=g6UWxK0iEVyBU3cOpvw9fiTYvOrC3uHRPFR1WTlWWRU,4583
 shancx/Resize.py,sha256=uwiOhD-aWPDLvIwOYEX-XXN_suOXv6hPDcaGyr4JLC4,4044
 shancx/ZR.py,sha256=5APchqVoI1G2MkrN9YJSWCwfLUygcSFIsUXSo9aq1Qg,341
 shancx/__init__.py,sha256=6sfiHElECqfy5eEEGqaCAw4YiKxgKfJ5IFFh9yvtewk,11442
@@ -18,7 +18,7 @@ shancx/getResponse.py,sha256=QmJfa4czGCOiKb8RuCLXKE7AAKVADAptNiwn7v1tfbM,1055
 shancx/info.py,sha256=0rk_L8Z5uj9RliaZrzT-CArH9ZObmdmcp-1RxAItL08,1122
 shancx/netdfJU.py,sha256=96KR9NMLr2Kcs_OOMpX7QPujdWUj0GCEUIZ7_4_pBxw,7741
 shancx/npz.py,sha256=ZydULlx1ZhgwxrdIrVDm9lGa8KY4q1MJ9dZo_wQriso,462
-shancx/sendM.py,sha256=CEpeHavseWILlfevhfOMYxYUIWhh4fCNkXGRsEjMxBE,3352
+shancx/sendM.py,sha256=dcqGNPPy1n0NxrRgOrHZEpaGtLy15FS_lCYe6_TkvSs,3510
 shancx/wait.py,sha256=OgT9JmEgMH7d3ir3TEksx0_Kyl_S4bkOKl50jvVpdFA,8575
 shancx/3D/__init__.py,sha256=pDFvW1OOEW-ueXH50NJIYdWlySHZU6sr5htzHUSeROo,1000
 shancx/Algo/Class.py,sha256=bxJ9JhIjoxl0hJfjO126YrnyHWDpxCEAZ7qcPs0Y0ic,377
@@ -63,9 +63,10 @@ shancx/H9/__init__.py,sha256=FCarcXfU2tVD2KrCGKNNUuL51zAWZYHV7lrP7gntvaI,4911
 shancx/H9/ahi_read_hsd.py,sha256=bt9oOOARcXijmyGpmHYXj0NKnuTntZjqx0_tu6Vp2vs,33522
 shancx/H9/ahisearchtable.py,sha256=e2kpz-P5npgL4gzNxn8igERJuWWIysvTNLkptr5_Zcc,9579
 shancx/H9/geometry.py,sha256=ZoCfgP07ANqDxyLf8uFN016DPl7QqW9pAV3sr2ekCng,95893
-shancx/Hug/__init__.py,sha256=NvcqXM0CSwXYtTr5yj4zfkag_7PZcI5tCsIHjuLNXTQ,3040
+shancx/Hug/__init__.py,sha256=uSPle6I70G6C6UzOs3OABiGqvlhxG3cr3xmP4kFStiE,3037
 shancx/Mos/__init__.py,sha256=IDlCLNN6a_mguYmRcAPcevHSDaud5ZhvYNSuASqztcE,1563
 shancx/NN/__init__.py,sha256=ENgZxIzUJzkuxArkcH-1JlxQA08Urq2ppFkGCbIURZk,9219
+shancx/NN/chainMul.py,sha256=p7q_jrgFQkkfAYr_ZJFirwZOYoyn31QOVYERwhBGH4k,8649
 shancx/Plot/GlobMap.py,sha256=WRvZOf5bo8IfECk1-sN4g6K0CrcJhSoEJAiLvZAWDwg,13021
 shancx/Plot/Gray2RGB.py,sha256=yDXu1xWVi7OtrS8_ExxK_ev1988kj1ZxoNCWf-kl4Zc,2734
 shancx/Plot/__init__.py,sha256=_fmDFZfh9HzDPBPKCMPrDT2DsblLVfrO-uFvUg76a74,19194
@@ -86,7 +87,7 @@ shancx/Train/multiGpu.py,sha256=D_oZeiSc7VWktpnVDwrFOC1CYZSt9rxOKY5lngE5vFg,820
 shancx/Train/prepare.py,sha256=vL_8UOA66oZCBIwCICtihsGibivtNgaVJGulJxfNdn8,6793
 shancx/Train/renet50.py,sha256=wEhYk1X96WE5zuqHqVxWLJa-A5jDNkz4z6edORNufnA,6428
 shancx/tensBoard/__init__.py,sha256=ga2C5YyJITvvQA1ocpxna_KNFnNRJVwkTjLoIglLZUQ,993
-shancx-1.9.33.220.dist-info/METADATA,sha256=B8qXJCc27DQM40dNoJCVp619BwyBKn7HAUnAJSdn44M,850
-shancx-1.9.33.220.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-shancx-1.9.33.220.dist-info/top_level.txt,sha256=akfCS1vKWz3pNmEN_yN9ZiGp-60IQY5ET38mRx_i_-4,7
-shancx-1.9.33.220.dist-info/RECORD,,
+shancx-1.9.33.226.dist-info/METADATA,sha256=sZuxCrfePOwXAfvoPyNJNqhaHW3MQcQAh-JDl-bn2Lo,850
+shancx-1.9.33.226.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+shancx-1.9.33.226.dist-info/top_level.txt,sha256=akfCS1vKWz3pNmEN_yN9ZiGp-60IQY5ET38mRx_i_-4,7
+shancx-1.9.33.226.dist-info/RECORD,,