shancx 1.9.33.170__tar.gz → 1.9.33.172__tar.gz

{shancx-1.9.33.170 → shancx-1.9.33.172}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shancx
-Version: 1.9.33.170
+Version: 1.9.33.172
 Summary: A simple timer decorator
 Home-page: https://gitee.com/shancx
 Author: shancx

{shancx-1.9.33.170 → shancx-1.9.33.172}/setup.py

@@ -3,7 +3,7 @@
 from setuptools import setup, find_packages
 setup(
     name="shancx",
-    version="1.9.33.170",
+    version="1.9.33.172",
     packages=find_packages(),
     description="A simple timer decorator",
     long_description=open("README.md").read(),

shancx-1.9.33.172/shancx/NN/__init__.py

@@ -0,0 +1,161 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+import os
+def start():
+    print("import successful")
+# constants
+__author__ = 'shancx' 
+__author_email__ = 'shanhe12@163.com'
+__time__ = '20251028 21:16'
+import logging
+from logging.handlers import RotatingFileHandler
+import os
+from shancx import crDir 
+def _loggers(logger_name="loggers", root="./logs", phase="project", level=logging.INFO, screen=True, max_bytes=10*1024*1024, backup_count=5, overwrite=False,handlersflag=False):
+    '''set up logger with rotating file handler'''
+    l = logging.getLogger(logger_name)
+    if handlersflag:
+        if l.handlers:
+            return l
+    formatter = logging.Formatter(
+        "%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s", datefmt='%Y-%m-%d %H:%M:%S') 
+    log_file = os.path.join(root, '{}.log'.format(phase))
+    crDir(log_file)
+    # Use RotatingFileHandler with 'w' mode to overwrite log file if needed
+    mode = 'w' if overwrite else 'a'
+    fh = RotatingFileHandler(log_file, mode=mode, maxBytes=max_bytes, backupCount=backup_count)
+    fh.setFormatter(formatter)    
+    l.setLevel(level)
+    l.addHandler(fh)    
+    if screen:
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        l.addHandler(sh)    
+    return l
+
+"""
+logger = _loggers(logger_name="test_logger", root=curpathplus, phase="test_log", overwrite=True, screen=True)
+# 测试日志输出
+for i in range(5):
+    logger.info(f"这是日志消息 {i+1}")
+    time.sleep(1)
+"""
+
+import logging
+def setlogger(level=logging.INFO):
+ 
+    logging.basicConfig(
+        level=level,  # 动态接受级别参数
+        format="%(asctime)s - %(filename)s[line:%(lineno)d] - %(levelname)s: %(message)s",
+        handlers=[logging.StreamHandler()],
+        force=True  # 强制覆盖现有配置（Python 3.8+）
+    )
+    return logging.getLogger()
+'''
+# 使用示例
+if __name__ == "__main__":
+    logger = setlogger(level=logging.DEBUG)  # 设置为DEBUG级别
+    logger.debug("这条日志会显示")  # 默认情况下DEBUG不显示，但因为我们设置了级别，现在会显示
+    logger.info("这是一条INFO日志")
+'''
+
+import os
+import glob
+import logging
+from datetime import datetime
+from shancx import loggers as logger
+def cleanupLogs(log_dir='/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs', keep_count=10, 
+                 pattern='*.log', recursive=False, dry_run=False):
+    stats = {'total_dirs': 0, 'deleted_files': 0, 'errors': []}    
+    def _cleanup_dir(directory):
+        stats['total_dirs'] += 1
+        if not os.path.exists(directory):
+            logging.warning(f"目录不存在: {directory}")
+            return
+        file_paths = glob.glob(os.path.join(directory, pattern))
+        log_files = [(path, os.path.getmtime(path)) for path in file_paths]
+        log_files.sort(key=lambda x: x[1], reverse=True)        
+        if len(log_files) <= keep_count:
+            logging.info(f"目录 {directory} 中的文件数量 ({len(log_files)}) 不超过保留数量 ({keep_count})，无需清理")
+            return
+        files_to_delete = log_files[keep_count:]
+        for file_path, mtime in files_to_delete:
+            try:
+                if dry_run:
+                    logging.info(f"[试运行] 将删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                else:
+                    os.remove(file_path)
+                    logging.info(f"已删除: {file_path} (修改时间: {datetime.fromtimestamp(mtime).strftime('%Y-%m-%d %H:%M:%S')})")
+                stats['deleted_files'] += 1
+            except Exception as e:
+                error_msg = f"删除失败 {file_path}: {str(e)}"
+                logging.error(error_msg)
+                stats['errors'].append(error_msg)
+    if recursive:
+        for root, _, _ in os.walk(log_dir):
+            _cleanup_dir(root)
+    else:
+        _cleanup_dir(log_dir)    
+    return stats
+"""
+if __name__ == "__main__": 
+    dir = "/mnt/wtx_weather_forecast/scx/SATH9SEAStest/logs/H9SEAS/"
+    stats = cleanupLogs(dir,3, '*.log', False, False)    
+    logging.info(f"清理完成: 处理目录数={stats['total_dirs']}, 删除文件数={stats['deleted_files']}, 错误数={len(stats['errors'])}")
+    if stats['errors']:
+        logging.error(f"错误详情: {stats['errors']}")
+"""  
+
+from itertools import product
+from concurrent.futures import ProcessPoolExecutor as PoolExecutor, as_completed 
+import sys
+from tqdm import tqdm 
+def validate_param_list(param_list):
+    if len(param_list) == 0:
+        raise ValueError("param_list cannot be empty.")    
+    for sublist in param_list:
+        if len(sublist) == 0:
+            raise ValueError("Sub-lists in param_list cannot be empty.")     
+def Mul_sub(task, param_list, num=6):
+    print(f"Pro num {num}")
+    validate_param_list(param_list)
+    if len(param_list) == 1:
+        product_list = [(x,) for x in param_list[0]]
+    else:
+        product_list = list(product(*param_list))
+    with PoolExecutor(max_workers=num) as executor:
+        try:           
+            futures = [executor.submit(task, item) for item in product_list]
+            for future in tqdm(as_completed(futures), total=len(futures), desc="Processing tasks", unit="task"):  
+                future.result() 
+        except KeyboardInterrupt:
+            sys.exit(1)    
+    print("All tasks completed")
+
+# import os
+# from datetime import datetime
+# from shancx.NN import _loggers
+# from shancx import lock_file
+# from shancx.wait import check_lock
+# from shancx import crDir
+# logger =_loggers()
+# def check_process_data(UTC, sat_cd,basepath ="/mnt/wtx_weather_forecast/scx/test/lock_files" ): 
+#     try:
+#         UTCStr = UTC.strftime("%Y%m%d%H%M")
+#         file = f"/mnt/wtx_weather_forecast/scx/test/lock_files/{sat_cd}/{UTCStr[:4]}/{UTCStr[:8]}/File_{UTCStr}.lock"
+#         crDir(file)
+#         if not lock_file(file):
+#             if check_lock(file):
+#                 logger.info("data is making or maked")
+#                 return True ,file       
+#         return False,file        
+#     except Exception as e:
+#         logger.error(f"Error in check_and_process_data: {str(e)}")
+#         return False,file  
+# """
+# flag1,file = check_process_data(UTC, "H9SEAS" )
+# if flag1:
+#     sys.exit() 
+# if os.path.exists(output_path): #配合使用
+#    sys.exit()    
+# """

{shancx-1.9.33.170 → shancx-1.9.33.172}/shancx.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.1
 Name: shancx
-Version: 1.9.33.170
+Version: 1.9.33.172
 Summary: A simple timer decorator
 Home-page: https://gitee.com/shancx
 Author: shancx

{shancx-1.9.33.170 → shancx-1.9.33.172}/shancx.egg-info/SOURCES.txt

@@ -64,10 +64,6 @@ shancx/Fillmiss/__init__.py
 shancx/Fillmiss/imgidwJU.py
 shancx/Fillmiss/imgidwLatLonJU.py
 shancx/Gpu/__init__.py
-shancx/H9/__init__.py
-shancx/H9/ahi_read_hsd.py
-shancx/H9/ahisearchtable.py
-shancx/H9/geometry.py
 shancx/Hug/__init__.py
 shancx/Mos/__init__.py
 shancx/NN/__init__.py

shancx-1.9.33.170/shancx/H9/__init__.py

@@ -1,126 +0,0 @@
-from shancx.H9.ahisearchtable import ahisearchtable
-from shancx.H9.ahi_read_hsd import ahi_read_hsd
-from tqdm import tqdm
-import os
-import numpy as np 
-import datetime 
-import time
-import warnings
-warnings.filterwarnings("ignore", category=UserWarning)
-class AHIScene(ahi_read_hsd, ahisearchtable) :
-    def __init__(self, subpoint=140.7, resolution=0.02):
-        super().__init__(subpoint=subpoint, resolution=resolution)
-        self.Tempfile = []
-    def hsdBlock(self, srcHSDfiles, tmppath, fillvalue=65535) :
-        ''' 对H8、H9的HSD文件进行解析、拼接成NOM '''
-        # HS_H09_20230115_0400_B01_FLDK_R10_S0110.DAT.bz2
-        BandID, BlockIDMin, BlockIDMax, SegmentTotal = self.setHSDInfo(srcHSDfiles)
-        outdata = None
-        BlockIDs = []
-        with tqdm(total=len(srcHSDfiles), iterable='iterable',
-                  desc = '正在进行第%i波段块合成' %(BandID), mininterval=1) as pbar:
-            for hsdname in srcHSDfiles :
-                if not os.path.isfile(hsdname):
-                    print('文件不存在【%s】' %(hsdname))
-                    pbar.update(1)
-                    continue
-
-                # 获取文件名信息
-                nameinfo = self.getHSDNameInfo(hsdname)
-                if nameinfo is None :
-                    pbar.update(1)
-                    continue
-                SegmentNum = nameinfo['SegmemtID']
-
-                # print('正在解压bz2文件【%s】' %(hsdname))
-                self._unzipped = self.unzip_file(hsdname, tmppath)
-                if self._unzipped:
-                    self.is_zipped = True
-                    filename = self._unzipped
-
-                    self.Tempfile.append(filename)
-                else:
-                    filename = hsdname
-
-                if filename.endswith('.bz2') :
-                    print('解压bz2文件失败【%s】' %(filename))
-                    pbar.update(1)
-                    continue
-
-                # 根据块号对数据进行拼接
-                data = self.readhsd(filename, SegmentNum)
-                if data is None :
-                    pbar.update(1)
-                    continue
-
-                if outdata is None :
-                    line, pixel = data.shape
-                    outdata = np.full(shape=(line*SegmentTotal, pixel),
-                                      fill_value=fillvalue, dtype=np.uint16)
-
-                data[np.isnan(data)] = fillvalue/100.0
-                outdata[(SegmentNum-BlockIDMin)*line:(SegmentNum-BlockIDMin+1)*line, :] \
-                    = np.array(data*100.0, dtype=np.uint16)
-                BlockIDs.append(SegmentNum)
-                pbar.update(1)
-        pbar.close()
-        self.__del__()
-        return outdata 
-  
-    def setHSDInfo(self, filelist):
-
-        BandID = None
-        BlockIDs = []
-        for filename in filelist :
-            nameinfo = self.getHSDNameInfo(filename)
-            if nameinfo is None :
-                continue
-
-            if BandID is None :
-                BandID = nameinfo['BandID']
-            elif BandID != nameinfo['BandID'] :
-                raise Exception('输入的文件列表中有多个波段的块数据文件【%s】' %(filename))
-            BlockIDs.append(nameinfo['SegmemtID'])
-
-        BlockIDMin = np.nanmin(BlockIDs)
-        BlockIDMax = np.nanmax(BlockIDs)
-
-        SegmentTotal = int(BlockIDMax-BlockIDMin+1)
-
-        return BandID, BlockIDMin, BlockIDMax, SegmentTotal
-
-    def getHSDNameInfo(self, filename):
-
-        basename = os.path.basename(filename)
-        basename = basename.split('.')[0]
-        if len(basename) != 39 :
-            print('非标准文件名，需要输入文件名【HS_H09_YYYYMMDD_HHMM_BXX_FLDK_R20_S0810】')
-            return None
-
-        nameinfo = {}
-        namelist = basename.split('_')
-
-        nameinfo['SatID'] = namelist[1]
-        nameinfo['StartTime'] = datetime.datetime.strptime('%s %s' %(namelist[2], namelist[3]), '%Y%m%d %H%M')
-        nameinfo['BandID'] = int(namelist[4][1:])   # 2-digit band number (varies from "01" to "16");
-        nameinfo['ObsType'] = namelist[5]
-        nameinfo['Resolution'] = float(namelist[6][1:])/10.0/100    # spatial resolution ("05": 0.5km, "10": 1.0km, "20": 2.0km);
-        nameinfo['SegmemtID'] = int(namelist[7][1:3])
-        nameinfo['SegmemtTotal'] = int(namelist[7][3:5])    # total number of segments (fixed to "10")
-
-        return nameinfo
-
-    def __del__(self):
-        # pass
-        for filename in self.Tempfile :
-            if os.path.isfile(filename) :
-                try:
-                    os.remove(filename)
-                except BaseException as e :
-                    time.sleep(1)
-                    try:
-                        fp = open(filename, 'r')
-                        fp.close()
-                        os.remove(filename)
-                    except BaseException as e :
-                        pass