shancx 1.8.92__py3-none-any.whl → 1.9.33.218__py3-none-any.whl

shancx/Diffm/test.py

@@ -0,0 +1,207 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import matplotlib.pyplot as plt
+import numpy as np
+from sklearn.datasets import make_s_curve
+import matplotlib.pyplot as plt
+from shancx import crDir
+s_curve, _ = make_s_curve(10**4, noise=0.1)
+s_curve = s_curve[:, [0, 2]] / 10.0
+dataset = torch.Tensor(s_curve).float()
+num_steps = 100
+# Define the beta schedule
+betas = torch.linspace(-6, 6, num_steps)
+betas = torch.sigmoid(betas) * (0.5e-2 - 1e-5) + 1e-5
+alphas = 1 - betas
+alphas_prod = torch.cumprod(alphas, 0)
+alphas_prod_p = torch.cat([torch.tensor([1]).float(), alphas_prod[:-1]], 0)
+alphas_bar_sqrt = torch.sqrt(alphas_prod)
+one_minus_alphas_bar_log = torch.log(1 - alphas_prod)
+one_minus_alphas_bar_sqrt = torch.sqrt(1 - alphas_prod)
+assert alphas.shape == alphas_prod.shape == alphas_prod_p.shape == \
+       alphas_bar_sqrt.shape == one_minus_alphas_bar_log.shape == \
+       one_minus_alphas_bar_sqrt.shape
+class MLPDiffusion(nn.Module):
+    def __init__(self, n_steps, num_units=128):
+        super(MLPDiffusion, self).__init__()
+        self.linears = nn.ModuleList([
+            nn.Linear(2, num_units),
+            nn.ReLU(),
+            nn.Linear(num_units, num_units),
+            nn.ReLU(),
+            nn.Linear(num_units, num_units),
+            nn.ReLU(),
+            nn.Linear(num_units, 2),
+        ])
+        self.step_embeddings = nn.ModuleList([
+            nn.Embedding(n_steps, num_units),
+            nn.Embedding(n_steps, num_units),
+            nn.Embedding(n_steps, num_units),
+        ])
+    
+    def forward(self, x, t):
+        for idx, embedding_layer in enumerate(self.step_embeddings):
+            t_embedding = embedding_layer(t)
+            x = self.linears[2 * idx](x)
+            x += t_embedding
+            x = self.linears[2 * idx + 1](x)
+        x = self.linears[-1](x)
+        return x
+import torch
+import torch.nn as nn
+import torch
+import torch.nn as nn
+class UNetDiffusion(nn.Module):
+    def __init__(self, n_steps, num_units=128):
+        super(UNetDiffusion, self).__init__()
+        self.encoder1 = nn.Sequential(
+            nn.Linear(2, num_units),
+            nn.ReLU(),
+            nn.Linear(num_units, num_units),
+            nn.ReLU()
+        )
+        self.encoder2 = nn.Sequential(
+            nn.Linear(num_units, num_units * 2),
+            nn.ReLU(),
+            nn.Linear(num_units * 2, num_units * 2),
+            nn.ReLU()
+        )
+        self.encoder3 = nn.Sequential(
+            nn.Linear(num_units * 2, num_units * 4),
+            nn.ReLU(),
+            nn.Linear(num_units * 4, num_units * 4),
+            nn.ReLU()
+        )
+        self.decoder3 = nn.Sequential(
+            nn.Linear(num_units * 4, num_units * 2),
+            nn.ReLU(),
+            nn.Linear(num_units * 2, num_units * 2),
+            nn.ReLU()
+        )
+        self.decoder2 = nn.Sequential(
+            nn.Linear(num_units * 2, num_units),
+            nn.ReLU(),
+            nn.Linear(num_units, num_units),
+            nn.ReLU()
+        )
+        self.decoder1 = nn.Sequential(
+            nn.Linear(num_units, 2), 
+        )
+        self.step_embeddings = nn.ModuleList([
+            nn.Embedding(n_steps, num_units),
+            nn.Embedding(n_steps, num_units),
+            nn.Embedding(n_steps, num_units),
+        ])
+
+    def forward(self, x, t):
+        # 对每个时间步 t，获取时间嵌入并将其加到输入特征中
+        for idx, embedding_layer in enumerate(self.step_embeddings):
+            t_embedding = embedding_layer(t)  # 获取当前时间步的嵌入向量
+            if idx == 0:
+                x = self.encoder1(x)
+            elif idx == 1:
+                x = self.encoder2(x)
+            else:
+                x = self.encoder3(x)
+            x += t_embedding  # 将时间步的嵌入加到当前层输出
+            if idx == 0:
+                x = self.encoder1(x)
+            elif idx == 1:
+                x = self.encoder2(x)
+            else:
+                x = self.encoder3(x)
+        # 解码器：逐层恢复
+        x = self.decoder3(x)
+        x = self.decoder2(x)
+        x = self.decoder1(x)
+        
+        return x
+def downsample_image(x, scale_factor=0.5, device=None):
+    result = x * scale_factor
+    return result
+def p_sample(model, x, t, betas, one_minus_alphas_bar_sqrt, device):
+    t = torch.tensor([t]).to(device)  # Move t to device
+    # betas = betas.to(device)
+    # one_minus_alphas_bar_sqrt = one_minus_alphas_bar_sqrt.to(device)
+
+    coeff = betas[t].to(device) / one_minus_alphas_bar_sqrt[t].to(device)
+    eps_theta = model(x, t)
+
+    mean = (1 / (1 - betas[t]).sqrt()) * (x - (coeff * eps_theta))
+    z = torch.randn_like(x).to(device)
+    sigma_t = betas[t].sqrt()
+
+    sample = mean + sigma_t * z
+    return sample 
+def p_sample_loop(model, shape, n_steps, betas, one_minus_alphas_bar_sqrt, device):
+    betas = betas.to(device)
+    one_minus_alphas_bar_sqrt = one_minus_alphas_bar_sqrt.to(device)
+    cur_x = torch.randn(shape).to(device)  # Move initial x to device
+    x_seq = [cur_x]
+    for i in reversed(range(n_steps)):
+        cur_x = p_sample(model, cur_x, i, betas, one_minus_alphas_bar_sqrt, device)
+        x_seq.append(cur_x)
+    return x_seq
+def diffusion_loss_fn(model, x_0, alphas_bar_sqrt, one_minus_alphas_bar_sqrt, n_steps, device):
+    x_0 = x_0.to(device)
+    alphas_bar_sqrt = alphas_bar_sqrt.to(device)
+    one_minus_alphas_bar_sqrt = one_minus_alphas_bar_sqrt.to(device)
+
+    batch_size = x_0.shape[0]
+    t = torch.randint(0, n_steps, size=(batch_size // 2,)).to(device)
+    t = torch.cat([t, n_steps - 1 - t], dim=0)
+    t = t.unsqueeze(-1)
+
+    a = alphas_bar_sqrt[t].to(device)
+    aml = one_minus_alphas_bar_sqrt[t].to(device)
+
+    e = torch.randn_like(x_0).to(device)
+    x = x_0 * a + e * aml
+
+    output = model(x, t.squeeze(-1))
+    return (e - output).square().mean()
+def super_resolution_loss(model, x_0, alphas_bar_sqrt, one_minus_alphas_bar_sqrt, n_steps, lr_scale=0.5, device=None):
+    x_0 = x_0.to(device)
+    lr_x_0 = downsample_image(x_0, scale_factor=lr_scale, device=device)
+    
+    loss = diffusion_loss_fn(model, lr_x_0, alphas_bar_sqrt, one_minus_alphas_bar_sqrt, n_steps, device=device).mean()
+    
+    reconstructed_x_0 = model(lr_x_0, torch.tensor([n_steps - 1]).to(device))
+    mse_loss = F.mse_loss(reconstructed_x_0, x_0).mean()
+
+    return (loss + mse_loss).mean()
+device = torch.device("cuda:1" if torch.cuda.is_available() else "cpu")
+# model = MLPDiffusion(n_steps=100).to(device)
+# optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
+model = UNetDiffusion(n_steps=100).to(device)
+optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
+# Training Loop
+num_epoch = 4000
+for t in range(num_epoch):
+    for idx, batch_x in enumerate(torch.utils.data.DataLoader(dataset, batch_size=2, shuffle=True)):
+        batch_x = batch_x.to(device)
+        loss = super_resolution_loss(model, batch_x, alphas_bar_sqrt, one_minus_alphas_bar_sqrt, num_steps, device=device)
+        optimizer.zero_grad()
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
+        optimizer.step()
+    if t % 100 == 0:
+        print(f'Epoch {t}, Loss: {loss.item()}')
+
+        # Sampling and visualization
+        x_seq = p_sample_loop(model, dataset.shape, num_steps, betas, one_minus_alphas_bar_sqrt, device)
+        
+        fig, axs = plt.subplots(1, 10, figsize=(28, 3))
+        for i in range(1, 11):            
+            if x_seq[i * 10] is not None:
+                cur_x = x_seq[i * 10].detach().cpu()
+                axs[i - 1].scatter(cur_x[:, 0], cur_x[:, 1], color='red', edgecolor='white')
+                axs[i - 1].set_axis_off()
+                axs[i - 1].set_title(f'$q(\\mathbf{{x}}_{{{i * 10}}})$')
+            else:
+                print(f"Warning: x_seq[{i * 10}] is None.")
+        plt.tight_layout()   
+        outpath = f'./pngresult/{t}_Epoch_scatter_plot.png'
+        crDir(outpath)
+        plt.savefig(outpath, dpi=300)   

shancx/Doc/__init__.py

@@ -0,0 +1,214 @@
+from docx import Document
+from docx.shared import Inches
+from docx.shared import Pt
+from functools import partial 
+def tableDOC(doc, data, row, col, title):
+    row_labels = row
+    col_labels = col
+    doc.add_paragraph(title)
+    table = doc.add_table(rows=1 + len(data), cols=1 + len(col_labels))
+    table.style = 'Table Grid'
+    header_cell = table.cell(0, 0)
+    header_cell.text = "实况\预报"
+    header_cell.paragraphs[0].runs[0].font.bold = False
+    for idx, label in enumerate(col_labels):
+        cell = table.cell(0, idx+1)
+        cell.text = label
+        cell.paragraphs[0].runs[0].font.bold = False
+    for row_idx, (label, row) in enumerate(zip(row_labels, data)):
+        table.cell(row_idx + 1, 0).text = label
+        table.cell(row_idx + 1, 0).paragraphs[0].runs[0].font.bold = False
+        for col_idx, value in enumerate(row):
+            table.cell(row_idx + 1, col_idx + 1).text = str(value) 
+def add_heading(doc, text):
+    heading = doc.add_heading(text, level=1)
+    heading.style.font.size = Pt(18)  # 设置字体大小为18磅
+def add_text_to_doc(doc, text):
+    para = doc.add_paragraph(text)
+    para.style = doc.styles['Body Text']
+def add_image_to_doc(doc, image_path):
+    doc.add_picture(image_path, width=Inches(5.0)) 
+def partialFN(tabledata):
+    return partial(tableDOC,row=tabledata["labels"], col= tabledata["columns"],title=tabledata["title"])
+
+dataCY ={
+    "labels": ["晴", "雨"],
+    "columns": ["晴", "雨"],
+    "title": f"彩云10min晴雨混淆矩阵"
+    } 
+dataWTX ={ 
+       "labels": ["晴", "雨"],
+       "columns": ["晴", "雨"],
+       "title": f"维天信10min晴雨混淆矩阵"
+       } 
+data2CY ={
+    "labels": ["晴", "雨"],
+    "columns": ["晴", "雨"],
+    "title": f"彩云20min晴雨混淆矩阵"
+    } 
+data2WTX ={ 
+       "labels": ["晴", "雨"],
+       "columns": ["晴", "雨"],
+       "title": f"维天信20min晴雨混淆矩阵"
+       } 
+
+# {0: ['0.016', '0.976', '6157.69%'], 1: ['0.177', '0.892', '405.15%']} dict_values([['0.386', '0.921', '138.33%'], ['0.399', '0.819', '105.52%']])  生成word表格输入数据
+
+# from docx import Document
+# doc = Document() 
+# doc.save("./test6.docx")
+
+"""
+
+import pandas as pd  
+from shancx.Dsalgor.matrixLibJU import TS ,ACC ,F1 ,FAR ,PO ,sun_rain_Matrix ,pre1h_Matrix
+from shancx.DOCJU import partialFN,add_text_to_doc,add_image_to_doc,add_heading
+import numpy as np
+
+from docx import Document
+
+
+dataCY ={
+    "labels": ["晴", "雨"],
+    "columns": ["晴", "雨"],
+    "title": f"彩云1H晴雨混淆矩阵"
+    } 
+dataWTX ={ 
+       "labels": ["晴", "雨"],
+       "columns": ["晴", "雨"],
+       "title": f"维天信1H晴雨混淆矩阵"
+       } 
+data2CY ={
+    "labels": ["晴", "雨"],
+    "columns": ["晴", "雨"],
+    "title": f"彩云2H晴雨混淆矩阵"
+    } 
+data2WTX ={ 
+       "labels": ["晴", "雨"],
+       "columns": ["晴", "雨"],
+       "title": f"维天信2H晴雨混淆矩阵"
+       } 
+
+TSVD= {
+       "labels": ["1h", "2h"],
+       "columns":["彩云", "维天信", "提升百分比"],
+       "title": "TS评分"
+      }
+F1VD = {
+        "labels": ["1h", "2h"],
+        "columns":["彩云", "维天信", "提升百分比"],
+        "title": "F1评分"
+      }
+ACCVD= {
+    "labels": ["1h", "2h"],
+    "columns":["彩云", "维天信", "提升百分比"],
+    "title": "准确率评分"
+    }
+POVD = {
+    "labels": ["1h", "2h"],
+    "columns":["彩云", "维天信", "提升百分比"],
+    "title": "漏报率评分 "                        
+    }
+FARVD = {
+    "labels": ["1h", "2h"],
+    "columns":["彩云", "维天信", "提升百分比"],
+    "title": "空报率评分 "
+     }
+
+w_1h_bigD = {
+           "labels": ["晴", "小雨", "中雨", "大雨", "暴雨"],
+           "columns": ["晴", "小雨", "中雨", "大雨", "暴雨"],
+           "title": f"维天信1小时预报"
+           }
+w_2h_bigD = {
+           "labels": ["晴", "小雨", "中雨", "大雨", "暴雨"],
+           "columns": ["晴", "小雨", "中雨", "大雨", "暴雨"],
+           "title": f"维天信2小时预报"
+           }
+  
+dataCY = partialFN(dataCY)
+dataWTX = partialFN(dataWTX)
+data2CY = partialFN(data2CY)
+data2WTX = partialFN(data2WTX)                                          
+TSVD = partialFN(TSVD)
+F1VD = partialFN(F1VD)
+ACCVD = partialFN(ACCVD)
+POVD = partialFN(POVD)
+FARVD = partialFN(FARVD)
+w_1h_bigD = partialFN(w_1h_bigD)
+w_2h_bigD = partialFN(w_2h_bigD)
+
+
+basepathcsv = "/mnt/wtx_weather_forecast/scx/EXAMDATACSV10min/10min_202412240000_202412250000.csv"
+df = pd.read_csv(basepathcsv) 
+TSV = TS(df["PRE1_r"], df["PRE1_w"], thresholdR=0.1, thresholdF=0.1) 
+
+TSV = {}  
+for i in range(1,3):
+    F1h =TS( df[f"PRE{i}_r"],df[f"PRE{i}_c"],thresholdR=0.1, thresholdF=0.031)
+    F1hm =TS(df[f"PRE{i}_r"],df[f"PRE{i}_w"],thresholdR=0.1, thresholdF=0.1)
+    TSV[i]=[np.round(F1h,3),np.round(F1hm,3),f"{np.round((F1hm-F1h)/F1h*100,2)}%"]
+    print(f"{i}h {np.round(F1h,3)} {np.round(F1hm,3)} {np.round((F1hm-F1h)/F1h*100,2)}%")
+print("F1")
+print("time", "CY", "WTX")
+F1V = {}
+for i in range(1,3):
+    F1h =F1(df[f"PRE{i}_r"], df[f"PRE{i}_c"],thresholdR=0.1, thresholdF=0.031)
+    F1hm =F1(df[f"PRE{i}_r"], df[f"PRE{i}_w"],thresholdR=0.1, thresholdF=0.1)
+    F1V[i] = [np.round(F1h, 3), np.round(F1hm, 3),f"{np.round((F1hm-F1h)/F1h*100,2)}%"]
+    print(f"{i}h {np.round(F1h,3)} {np.round(F1hm,3)} {np.round((F1hm-F1h)/F1h*100,2)}%")
+print("ACC")
+print("time", "CY", "WTX")
+ACCV = {}
+for i in range(1, 3):
+    F1h = ACC(df[f"PRE{i}_r"], df[f"PRE{i}_c"],thresholdR=0.1, thresholdF=0.031)
+    F1hm = ACC(df[f"PRE{i}_r"], df[f"PRE{i}_w"],thresholdR=0.1, thresholdF=0.1)
+    ACCV[i] = [np.round(F1h, 3), np.round(F1hm, 3),f"{np.round((F1hm-F1h)/F1h*100,2)}%"]
+    print(f"{i}h {np.round(F1h, 3)} {np.round(F1hm, 3)} {np.round((F1hm - F1h) / F1h * 100, 2)}%")
+print("PO")
+print("time", "CY", "WTX")
+POV = {}
+for i in range(1,3):
+    F1h =PO(df[f"PRE{i}_r"], df[f"PRE{i}_c"],thresholdR=0.1, thresholdF=0.031)
+    F1hm =PO(df[f"PRE{i}_r"], df[f"PRE{i}_w"],thresholdR=0.1, thresholdF=0.1)
+    POV[i] = [np.round(F1h, 3), np.round(F1hm, 3),f"{np.round((F1hm-F1h)/F1h*100,2)*-1}%"]
+    print(f"{i}h {np.round(F1h,3)} {np.round(F1hm,3)} {np.round((F1hm-F1h)/F1h*100,2)*-1}%")
+print("FAR")
+print("time", "CY", "WTX")
+FARV = {}
+for i in range(1, 3):
+    F1h = FAR(df[f"PRE{i}_r"], df[f"PRE{i}_c"],thresholdR=0.1, thresholdF=0.031)
+    F1hm = FAR(df[f"PRE{i}_r"], df[f"PRE{i}_w"],thresholdR=0.1, thresholdF=0.1)
+    FARV[i] = [np.round(F1h, 3), np.round(F1hm, 3),f"{np.round((F1hm-F1h)/F1h*100,2)*-1}%"]
+    print(f"{i}h {np.round(F1h, 3)}  {np.round(F1hm, 3)} {np.round((F1hm-F1h)/F1h*100,2)*-1}%")
+
+cm1_C = sun_rain_Matrix(df["PRE1_r"].values, df["PRE1_c"].values,0.031)
+cm1_W = sun_rain_Matrix(df["PRE1_r"].values, df["PRE1_w"].values,0.1)
+cm2_C = sun_rain_Matrix(df["PRE2_r"].values, df["PRE2_c"].values,0.031)
+cm2_W = sun_rain_Matrix(df["PRE2_r"].values, df["PRE2_w"].values,0.1)
+
+cm1_W_pre1h = pre1h_Matrix(df["PRE1_r"].values, df["PRE1_w"].values,"WTX")
+cm2_W_pre1h = pre1h_Matrix(df["PRE2_r"].values, df["PRE2_w"].values,"WTX")
+
+doc = Document()    #CY_fn(doc,data =
+add_heading(doc,"测试报告样例")
+dataCY   (doc,data = cm1_C)
+dataWTX  (doc,data =cm1_W )
+data2CY  (doc,data = cm2_C)
+data2WTX (doc,data = cm2_W)
+TSVD     (doc,data =TSV.values() )
+F1VD     (doc,data =F1V.values() )
+ACCVD    (doc,data = ACCV.values())
+add_image_to_doc(doc,"./TSF1.png")
+add_text_to_doc(doc,"TS评分、F1和准确率ACC对比")
+add_image_to_doc(doc,"./POFAR.png")
+add_text_to_doc(doc,"漏报PO、空报FAR对比")
+POVD     (doc,data = POV.values())
+FARVD    (doc,data = FARV.values())
+w_1h_bigD(doc,data = cm1_W_pre1h)   
+w_2h_bigD(doc,data = cm2_W_pre1h)
+add_image_to_doc(doc,"bigsmall.png")
+doc.save("./makedoc_test.docx")
+ 
+
+"""