oafuncs 0.0.78__py2.py3-none-any.whl → 0.0.79__py2.py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- oafuncs/oa_down/User_Agent-list.txt +13 -2837
- oafuncs/oa_down/hycom_3hourly.py +120 -42
- oafuncs/oa_down/test.py +52 -28
- oafuncs/oa_draw.py +45 -1
- {oafuncs-0.0.78.dist-info → oafuncs-0.0.79.dist-info}/METADATA +1 -1
- {oafuncs-0.0.78.dist-info → oafuncs-0.0.79.dist-info}/RECORD +9 -9
- {oafuncs-0.0.78.dist-info → oafuncs-0.0.79.dist-info}/LICENSE.txt +0 -0
- {oafuncs-0.0.78.dist-info → oafuncs-0.0.79.dist-info}/WHEEL +0 -0
- {oafuncs-0.0.78.dist-info → oafuncs-0.0.79.dist-info}/top_level.txt +0 -0
oafuncs/oa_down/hycom_3hourly.py
CHANGED
@@ -18,13 +18,15 @@ import os
|
|
18
18
|
import random
|
19
19
|
import time
|
20
20
|
import warnings
|
21
|
-
from concurrent.futures import ThreadPoolExecutor
|
21
|
+
from concurrent.futures import ThreadPoolExecutor, as_completed
|
22
22
|
from pathlib import Path
|
23
|
+
from threading import Lock
|
23
24
|
|
24
25
|
import matplotlib.pyplot as plt
|
25
26
|
import numpy as np
|
26
27
|
import pandas as pd
|
27
28
|
import requests
|
29
|
+
from bs4 import BeautifulSoup
|
28
30
|
from rich import print
|
29
31
|
from rich.progress import Progress
|
30
32
|
|
@@ -512,10 +514,13 @@ def get_ua():
|
|
512
514
|
# 去掉换行符和空行
|
513
515
|
ua_list = [line.strip() for line in ua_list if line.strip()]
|
514
516
|
|
517
|
+
# if current_platform == 'Linux':
|
518
|
+
# ua_list = [line for line in ua_list if 'Linux' in line]
|
519
|
+
|
515
520
|
return random.choice(ua_list)
|
516
521
|
|
517
522
|
|
518
|
-
def
|
523
|
+
def get_proxy_file():
|
519
524
|
# 获取当前脚本的绝对路径
|
520
525
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
521
526
|
# 构建ip.txt的绝对路径
|
@@ -531,15 +536,62 @@ def get_proxy():
|
|
531
536
|
return proxies
|
532
537
|
|
533
538
|
|
534
|
-
def
|
539
|
+
def scrape_and_categorize_proxies(choose_protocol="http"):
|
540
|
+
url = "https://topproxylinks.com/"
|
541
|
+
# 发送HTTP请求获取网页内容
|
542
|
+
response = requests.get(url)
|
543
|
+
# 使用BeautifulSoup解析网页
|
544
|
+
soup = BeautifulSoup(response.text, "html.parser")
|
545
|
+
|
546
|
+
# 初始化字典来存储不同协议的代理
|
547
|
+
proxies_dict = {"http": [], "socks4": [], "socks5": []}
|
548
|
+
|
549
|
+
# 查找表格中的所有行
|
550
|
+
tbody = soup.find("tbody")
|
551
|
+
|
552
|
+
if tbody:
|
553
|
+
for row in tbody.find_all("tr"):
|
554
|
+
# 提取协议、代理和国家的单元格
|
555
|
+
cells = row.find_all("td")
|
556
|
+
protocol = cells[0].text.strip().lower()
|
557
|
+
proxy = cells[1].text.strip()
|
558
|
+
|
559
|
+
# 根据协议分类存储代理
|
560
|
+
if protocol in proxies_dict:
|
561
|
+
proxies_dict[protocol].append(proxy)
|
562
|
+
|
563
|
+
if choose_protocol in proxies_dict:
|
564
|
+
proxies_list = proxies_dict[choose_protocol]
|
565
|
+
else:
|
566
|
+
proxies_list = proxies_dict["http"]
|
567
|
+
|
568
|
+
return proxies_list
|
569
|
+
|
570
|
+
def get_proxy():
|
571
|
+
ip_list = scrape_and_categorize_proxies(choose_protocol="http")
|
572
|
+
choose_ip = random.choice(ip_list)
|
573
|
+
proxies = {"http": f"http://{choose_ip}", "https": f"http://{choose_ip}"}
|
574
|
+
print(f'Using proxy: {proxies}')
|
575
|
+
return proxies
|
576
|
+
|
577
|
+
|
578
|
+
def download_file(target_url, store_path, file_name, check=False):
|
579
|
+
# Check if the file exists
|
580
|
+
fname = Path(store_path) / file_name
|
581
|
+
if check:
|
582
|
+
if check_existing_file(fname):
|
583
|
+
count_dict["skip"] += 1
|
584
|
+
return
|
585
|
+
clear_existing_file(fname)
|
586
|
+
|
587
|
+
# -----------------------------------------------
|
535
588
|
print(f"[bold #f0f6d0]Requesting {file_name}...")
|
536
589
|
# 创建会话
|
537
590
|
s = requests.Session()
|
538
591
|
download_success = False
|
539
592
|
request_times = 0
|
540
|
-
filename = Path(store_path) / file_name
|
541
593
|
|
542
|
-
def calculate_wait_time(time_str):
|
594
|
+
def calculate_wait_time(time_str, target_url):
|
543
595
|
import re
|
544
596
|
|
545
597
|
# 定义正则表达式,匹配YYYYMMDDHH格式的时间
|
@@ -561,16 +613,15 @@ def dlownload_file(target_url, store_path, file_name, check=False):
|
|
561
613
|
delta_t = delta_t / 3 + 1
|
562
614
|
else:
|
563
615
|
delta_t = 1
|
616
|
+
# 单个要素最多等待5分钟,不宜太短,太短可能请求失败;也不宜太长,太长可能会浪费时间
|
617
|
+
num_var = int(target_url.count("var="))
|
618
|
+
if num_var <= 0:
|
619
|
+
num_var = 1
|
620
|
+
return int(delta_t * 5 * 60 * num_var)
|
564
621
|
|
565
|
-
|
622
|
+
max_timeout = calculate_wait_time(file_name, target_url)
|
623
|
+
print(f"[bold #912dbc]Max timeout: {max_timeout} seconds")
|
566
624
|
|
567
|
-
max_timeout = calculate_wait_time(file_name)
|
568
|
-
|
569
|
-
if check:
|
570
|
-
if check_existing_file(filename):
|
571
|
-
count_dict['skip'] += 1
|
572
|
-
return
|
573
|
-
clear_existing_file(filename)
|
574
625
|
# print(f'Download_start_time: {datetime.datetime.now()}')
|
575
626
|
download_time_s = datetime.datetime.now()
|
576
627
|
order_list = ["1st", "2nd", "3rd", "4th", "5th", "6th", "7th", "8th", "9th", "10th"]
|
@@ -592,12 +643,16 @@ def dlownload_file(target_url, store_path, file_name, check=False):
|
|
592
643
|
# 保存文件
|
593
644
|
with open(filename, 'wb') as f:
|
594
645
|
f.write(response.content) """
|
595
|
-
|
596
|
-
|
646
|
+
|
647
|
+
if find_proxy:
|
648
|
+
proxies = get_proxy()
|
649
|
+
response = s.get(target_url, headers=headers, proxies=proxies, stream=True, timeout=random.randint(5, max_timeout))
|
650
|
+
else:
|
651
|
+
response = s.get(target_url, headers=headers, stream=True, timeout=random.randint(5, max_timeout)) # 启用流式传输
|
597
652
|
response.raise_for_status() # 如果请求返回的不是200,将抛出HTTPError异常
|
598
653
|
|
599
654
|
# 保存文件
|
600
|
-
with open(
|
655
|
+
with open(fname, "wb") as f:
|
601
656
|
print(f"[bold #96cbd7]Downloading {file_name}...")
|
602
657
|
for chunk in response.iter_content(chunk_size=1024):
|
603
658
|
if chunk:
|
@@ -605,12 +660,12 @@ def dlownload_file(target_url, store_path, file_name, check=False):
|
|
605
660
|
|
606
661
|
f.close()
|
607
662
|
|
608
|
-
# print(f'\r文件 {
|
609
|
-
if os.path.exists(
|
663
|
+
# print(f'\r文件 {fname} 下载成功', end="")
|
664
|
+
if os.path.exists(fname):
|
610
665
|
download_success = True
|
611
666
|
download_time_e = datetime.datetime.now()
|
612
667
|
download_delta = download_time_e - download_time_s
|
613
|
-
print(f"[#3dfc40]File [bold #dfff73]{
|
668
|
+
print(f"[#3dfc40]File [bold #dfff73]{fname} [#3dfc40]has been downloaded successfully, Time: [#39cbdd]{download_delta}")
|
614
669
|
count_dict["success"] += 1
|
615
670
|
# print(f'Download_end_time: {datetime.datetime.now()}')
|
616
671
|
|
@@ -716,7 +771,7 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
|
|
716
771
|
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}.nc"
|
717
772
|
if download_time_end is not None:
|
718
773
|
file_name = f"HYCOM_{variable_info[var]['var_name']}_{download_time}_{download_time_end}.nc"
|
719
|
-
|
774
|
+
download_file(submit_url, store_path, file_name, check)
|
720
775
|
else:
|
721
776
|
varlist = [_ for _ in var]
|
722
777
|
for key, value in var_group.items():
|
@@ -740,7 +795,7 @@ def prepare_url_to_download(var, lon_min=0, lon_max=359.92, lat_min=-80, lat_max
|
|
740
795
|
file_name = f"HYCOM_{key}_{download_time}.nc"
|
741
796
|
if download_time_end is not None:
|
742
797
|
file_name = f"HYCOM_{key}_{download_time}_{download_time_end}.nc"
|
743
|
-
|
798
|
+
download_file(submit_url, store_path, file_name, check)
|
744
799
|
|
745
800
|
|
746
801
|
def convert_full_name_to_short_name(full_name):
|
@@ -767,6 +822,22 @@ def download_task(var, time_str, time_str_end, lon_min, lon_max, lat_min, lat_ma
|
|
767
822
|
prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, time_str, time_str_end, depth, level, store_path, dataset_name, version_name, check)
|
768
823
|
|
769
824
|
|
825
|
+
def done_callback(future, progress, task, total, counter_lock):
|
826
|
+
"""
|
827
|
+
# 并行下载任务的回调函数
|
828
|
+
# 这个函数是为了并行下载而设置的,是必须的,直接调用direct_download并行下载会出问题
|
829
|
+
|
830
|
+
回调函数:当一个任务完成后,会调用这个函数,这样可以及时更新进度条,显示任务的完成情况。
|
831
|
+
本情况下,done_callback函数的作用是当一个任务完成后,更新进度条的进度,显示任务的完成情况。
|
832
|
+
这样,即使多个任务同时执行,也可以及时看到每个任务的完成情况,不会等到所有任务都完成才显示。
|
833
|
+
"""
|
834
|
+
|
835
|
+
global parallel_counter
|
836
|
+
with counter_lock:
|
837
|
+
parallel_counter += 1
|
838
|
+
progress.update(task, advance=1, description=f"[cyan]Downloading... {parallel_counter}/{total}")
|
839
|
+
|
840
|
+
|
770
841
|
def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min=-80, lat_max=90, depth=None, level=None, store_path=None, dataset_name=None, version_name=None, num_workers=None, check=False, ftimes=1):
|
771
842
|
"""
|
772
843
|
Description:
|
@@ -791,6 +862,10 @@ def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min
|
|
791
862
|
None
|
792
863
|
"""
|
793
864
|
ymdh_time_s, ymdh_time_e = str(time_s), str(time_e)
|
865
|
+
if num_workers is not None and num_workers > 1: # 如果使用多线程下载,用于进度条显示
|
866
|
+
global parallel_counter
|
867
|
+
parallel_counter = 0
|
868
|
+
counter_lock = Lock() # 创建一个锁,线程安全的计数器
|
794
869
|
if ymdh_time_s == ymdh_time_e:
|
795
870
|
prepare_url_to_download(var, lon_min, lon_max, lat_min, lat_max, ymdh_time_s, None, depth, level, store_path, dataset_name, version_name)
|
796
871
|
elif int(ymdh_time_s) < int(ymdh_time_e):
|
@@ -808,8 +883,10 @@ def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min
|
|
808
883
|
# 并行方式
|
809
884
|
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
810
885
|
futures = [executor.submit(download_task, var, time_str, None, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for time_str in time_list]
|
811
|
-
for i, future in enumerate(futures):
|
812
|
-
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]Downloading... {i+1}/{len(time_list)}"))
|
886
|
+
""" for i, future in enumerate(futures):
|
887
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]Downloading... {i+1}/{len(time_list)}")) """
|
888
|
+
for feature in as_completed(futures):
|
889
|
+
done_callback(feature, progress, task, len(time_list), counter_lock)
|
813
890
|
else:
|
814
891
|
new_time_list = get_time_list(ymdh_time_s, ymdh_time_e, 3 * ftimes, "hour")
|
815
892
|
total_num = len(new_time_list)
|
@@ -824,8 +901,10 @@ def download_hourly_func(var, time_s, time_e, lon_min=0, lon_max=359.92, lat_min
|
|
824
901
|
# 并行方式
|
825
902
|
with ThreadPoolExecutor(max_workers=num_workers) as executor:
|
826
903
|
futures = [executor.submit(download_task, var, new_time_list[i], time_list[int(min(len(time_list) - 1, int(i * ftimes + ftimes - 1)))], lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, check) for i in range(total_num)]
|
827
|
-
for i, future in enumerate(futures):
|
828
|
-
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]Downloading... {i+1}/{total_num}"))
|
904
|
+
""" for i, future in enumerate(futures):
|
905
|
+
future.add_done_callback(lambda _: progress.update(task, advance=1, description=f"[cyan]Downloading... {i+1}/{total_num}")) """
|
906
|
+
for feature in as_completed(futures):
|
907
|
+
done_callback(feature, progress, task, len(time_list), counter_lock)
|
829
908
|
else:
|
830
909
|
print("Please ensure the time_s is no more than time_e")
|
831
910
|
|
@@ -907,31 +986,30 @@ def download(var, time_s, time_e=None, lon_min=0, lon_max=359.92, lat_min=-80, l
|
|
907
986
|
time_e = str(time_e)
|
908
987
|
if len(time_e) == 8:
|
909
988
|
time_e += "21"
|
910
|
-
|
989
|
+
|
911
990
|
global count_dict
|
912
|
-
count_dict = {
|
913
|
-
'success': 0,
|
914
|
-
'fail': 0,
|
915
|
-
'skip': 0,
|
916
|
-
'no_data': 0,
|
917
|
-
'total': 0,
|
918
|
-
'no_data_list': []
|
919
|
-
}
|
991
|
+
count_dict = {"success": 0, "fail": 0, "skip": 0, "no_data": 0, "total": 0, "no_data_list": []}
|
920
992
|
|
921
|
-
|
993
|
+
""" global current_platform
|
994
|
+
current_platform = platform.system() """
|
922
995
|
|
996
|
+
global find_proxy
|
997
|
+
find_proxy = False
|
998
|
+
|
999
|
+
download_hourly_func(var, time_s, time_e, lon_min, lon_max, lat_min, lat_max, depth, level, store_path, dataset_name, version_name, num_workers, check, ftimes)
|
1000
|
+
|
923
1001
|
count_dict["total"] = count_dict["success"] + count_dict["fail"] + count_dict["skip"] + count_dict["no_data"]
|
924
|
-
|
1002
|
+
|
925
1003
|
print("[bold #ecdbfe]-" * 160)
|
926
1004
|
print(f"[bold #ff80ab]Total: {count_dict['total']}\nSuccess: {count_dict['success']}\nFail: {count_dict['fail']}\nSkip: {count_dict['skip']}")
|
927
|
-
if count_dict[
|
1005
|
+
if count_dict["fail"] > 0:
|
928
1006
|
print("[bold #be5528]Please try again to download the failed data later")
|
929
|
-
if count_dict[
|
930
|
-
if count_dict[
|
1007
|
+
if count_dict["no_data"] > 0:
|
1008
|
+
if count_dict["no_data"] == 1:
|
931
1009
|
print(f"[bold #f90000]There is {count_dict['no_data']} data that does not exist in any dataset and version")
|
932
1010
|
else:
|
933
1011
|
print(f"[bold #f90000]These are {count_dict['no_data']} data that do not exist in any dataset and version")
|
934
|
-
for no_data in count_dict[
|
1012
|
+
for no_data in count_dict["no_data_list"]:
|
935
1013
|
print(f"[bold #d81b60]{no_data}")
|
936
1014
|
print("[bold #ecdbfe]-" * 160)
|
937
1015
|
|
@@ -988,7 +1066,7 @@ def how_to_use():
|
|
988
1066
|
if __name__ == "__main__":
|
989
1067
|
# help(hycom3h.download)
|
990
1068
|
time_s, time_e = "2018070100", "2019123121"
|
991
|
-
merge_name = "
|
1069
|
+
merge_name = f"{time_s}_{time_e}" # 合并后的文件名
|
992
1070
|
root_path = r"G:\Data\HYCOM\3hourly"
|
993
1071
|
location_dict = {"west": 105, "east": 130, "south": 15, "north": 45}
|
994
1072
|
download_dict = {
|
@@ -1012,7 +1090,7 @@ if __name__ == "__main__":
|
|
1012
1090
|
# if you wanna download all depth or level, set both False
|
1013
1091
|
depth = None # or 0-5000 meters
|
1014
1092
|
level = None # or 1-40 levels
|
1015
|
-
num_workers =
|
1093
|
+
num_workers = 3
|
1016
1094
|
|
1017
1095
|
check = True
|
1018
1096
|
ftimes = 1
|
oafuncs/oa_down/test.py
CHANGED
@@ -1,45 +1,45 @@
|
|
1
1
|
#!/usr/bin/env python
|
2
2
|
# coding=utf-8
|
3
|
-
|
3
|
+
"""
|
4
4
|
Author: Liu Kun && 16031215@qq.com
|
5
5
|
Date: 2024-12-01 19:32:25
|
6
6
|
LastEditors: Liu Kun && 16031215@qq.com
|
7
|
-
LastEditTime: 2024-12-
|
7
|
+
LastEditTime: 2024-12-10 11:16:36
|
8
8
|
FilePath: \\Python\\My_Funcs\\OAFuncs\\oafuncs\\oa_down\\test.py
|
9
|
-
Description:
|
9
|
+
Description:
|
10
10
|
EditPlatform: vscode
|
11
11
|
ComputerInfo: XPS 15 9510
|
12
12
|
SystemInfo: Windows 11
|
13
13
|
Python Version: 3.12
|
14
|
-
|
14
|
+
"""
|
15
15
|
|
16
16
|
import os
|
17
17
|
import random
|
18
|
+
import re
|
18
19
|
|
19
|
-
txtfile = r'E:\Code\Python\My_Funcs\OAFuncs\oafuncs\oa_down\User_Agent-list.txt'
|
20
20
|
|
21
|
-
|
22
|
-
|
23
|
-
#
|
24
|
-
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
print(lines[i])
|
21
|
+
def is_valid_user_agent(user_agent):
|
22
|
+
# 简单的正则表达式来检查User Agent的格式
|
23
|
+
# 这个正则表达式检查User Agent是否包含常见的浏览器信息格式
|
24
|
+
pattern = re.compile(
|
25
|
+
r"^(?:(?:Mozilla|Opera|Chrome|Safari|Edg|OPR)/[\d.]+)"
|
26
|
+
r"(?:\s(?:\(.*?\)))?"
|
27
|
+
r"(?:\s(?:Gecko|AppleWebKit|KHTML, like Gecko|Version|Edge|OPR)/[\d.]+)?"
|
28
|
+
r"(?:\s.*?(?:rv:|Version/|Ubuntu|Macintosh|Windows|X11|Linux|CrOS|FreeBSD|OpenBSD|NetBSD|iPhone|iPad|iPod|Android|BlackBerry|BB10|Mobile|Symbian|Windows Phone|IEMobile|Opera Mini|Opera Mobi|UCBrowser|MQQBrowser|baiduboxapp|baidubrowser|Safari|Firefox|MSIE|Trident|Edge|EdgA|Chrome|CriOS|Vivaldi|Sleipnir|Midori|ELinks|Lynx|w3m|Arora|Epiphany|Konqueror|Dillo|Netscape|SeaMonkey|K-Meleon|Camino|Iceape|Galeon|GranParadiso|Iceweasel|Firefox|Fennec|Conkeror|PaleMoon|Uzbl|QupZilla|Otter|Waterfox|Basilisk|Cyberfox|PaleMoon|GNU IceCat|GNU IceWeasel|IceCat|IceWeasel|Seamonkey|Iceape|Firefox|Epiphany|Web|Safari|Android|Mobile|BlackBerry|BB10|Tablet|Silk|Kindle|FxiOS|Focus|SamsungBrowser|browser|AppleWebKit|Puffin|DuckDuckGo|YaBrowser|Yandex|Amigo|NokiaBrowser|OviBrowser|OneBrowser|Chrome|Firefox|Safari|OPR|Coast|Mercury|Silk|Skyfire|IEMobile|Bolt|Jasmine|NativeHost|Crosswalk|TizenBrowser|SailfishBrowser|SamsungBrowser|Silk-Accelerated|UCBrowser|Quark|XiaoMi|OnePlus|Vivo|Oppo|Realme|Meizu|Lenovo|Huawei|ZTE|Alcatel|Sony|Nokia|LG|HTC|Asus|Acer|Motorola|Samsung)/[\d.]+)?$"
|
29
|
+
)
|
31
30
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
31
|
+
# 使用正则表达式匹配User Agent字符串
|
32
|
+
if pattern.match(user_agent):
|
33
|
+
return True
|
34
|
+
else:
|
35
|
+
return False
|
36
36
|
|
37
37
|
|
38
38
|
def get_ua():
|
39
39
|
current_dir = os.path.dirname(os.path.abspath(__file__))
|
40
|
-
ua_file_txt = os.path.join(current_dir,
|
40
|
+
ua_file_txt = os.path.join(current_dir, "User_Agent-list.txt")
|
41
41
|
|
42
|
-
with open(ua_file_txt,
|
42
|
+
with open(ua_file_txt, "r") as f:
|
43
43
|
ua_list = f.readlines()
|
44
44
|
# 去掉换行符和空行
|
45
45
|
ua_list = [line.strip() for line in ua_list if line.strip()]
|
@@ -47,9 +47,6 @@ def get_ua():
|
|
47
47
|
return random.choice(ua_list)
|
48
48
|
|
49
49
|
|
50
|
-
print(get_ua())
|
51
|
-
|
52
|
-
|
53
50
|
def get_ua_org():
|
54
51
|
ua_list = [
|
55
52
|
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/39.0.2171.95 Safari/537.36 OPR/26.0.1656.60",
|
@@ -115,13 +112,40 @@ def get_ua_org():
|
|
115
112
|
"NOKIA5700/UCWEB7.0.2.37/28/999",
|
116
113
|
"Openwave/UCWEB7.0.2.37/28/999",
|
117
114
|
"Openwave/UCWEB7.0.2.37/28/999",
|
118
|
-
|
119
115
|
]
|
120
|
-
with open(newtxtfile,
|
116
|
+
with open(newtxtfile, "w") as f:
|
121
117
|
for line in ua_list:
|
122
|
-
f.write(line +
|
118
|
+
f.write(line + "\n")
|
123
119
|
# print(f'Using User-Agent: {ua}')
|
124
120
|
ua = random.choice(ua_list)
|
125
121
|
return ua
|
126
122
|
|
127
|
-
|
123
|
+
|
124
|
+
# get_ua_org()
|
125
|
+
|
126
|
+
if __name__ == "__main__":
|
127
|
+
txtfile = r"E:\Code\Python\My_Funcs\OAFuncs\oafuncs\oa_down\User_Agent-list.txt"
|
128
|
+
|
129
|
+
with open(txtfile, "r") as f:
|
130
|
+
lines = f.readlines()
|
131
|
+
# 去掉换行符和空行
|
132
|
+
lines = [line.strip() for line in lines if line.strip()]
|
133
|
+
""" new_line = []
|
134
|
+
for i in range(len(lines)):
|
135
|
+
if '/' in lines[i]:
|
136
|
+
new_line.append(lines[i])
|
137
|
+
else:
|
138
|
+
print(lines[i]) """
|
139
|
+
|
140
|
+
new_line = []
|
141
|
+
for line in lines:
|
142
|
+
if is_valid_user_agent(line):
|
143
|
+
# print(line)
|
144
|
+
new_line.append(line)
|
145
|
+
else:
|
146
|
+
print(f"Invalid User-Agent: {line}")
|
147
|
+
|
148
|
+
newtxtfile = r"E:\Code\Python\My_Funcs\OAFuncs\oafuncs\oa_down\ua_list_new.txt"
|
149
|
+
with open(newtxtfile, "w") as f:
|
150
|
+
for line in new_line:
|
151
|
+
f.write(line + "\n")
|
oafuncs/oa_draw.py
CHANGED
@@ -26,10 +26,54 @@ import xarray as xr
|
|
26
26
|
from cartopy.mpl.ticker import LatitudeFormatter, LongitudeFormatter
|
27
27
|
from mpl_toolkits.axes_grid1 import make_axes_locatable
|
28
28
|
|
29
|
-
__all__ = ['create_gif', 'xy2lonlat', 'plot_contourf', 'plot_contourf_lonlat', 'plot_quiver', 'plot_contourf_cartopy']
|
29
|
+
__all__ = ['fig_minus', 'create_gif', 'xy2lonlat', 'plot_contourf', 'plot_contourf_lonlat', 'plot_quiver', 'plot_contourf_cartopy']
|
30
30
|
|
31
31
|
warnings.filterwarnings('ignore')
|
32
32
|
|
33
|
+
def fig_minus(ax_x=None, ax_y=None, cbar=None, decimal=None, add_space=False):
|
34
|
+
'''
|
35
|
+
Description: 将坐标轴刻度中的负号替换为减号
|
36
|
+
|
37
|
+
param {*} ax_x : x轴
|
38
|
+
param {*} ax_y : y轴
|
39
|
+
param {*} cbar : colorbar
|
40
|
+
param {*} decimal : 小数位数
|
41
|
+
param {*} add_space : 是否在非负数前面加空格
|
42
|
+
|
43
|
+
return {*} ax_x or ax_y or cbar
|
44
|
+
'''
|
45
|
+
if ax_x is not None:
|
46
|
+
current_ticks = ax_x.get_xticks()
|
47
|
+
if ax_y is not None:
|
48
|
+
current_ticks = ax_y.get_yticks()
|
49
|
+
if cbar is not None:
|
50
|
+
current_ticks = cbar.get_ticks()
|
51
|
+
# 先判断是否需要加空格,如果要,先获取需要加的索引
|
52
|
+
if add_space:
|
53
|
+
index = 0
|
54
|
+
for _, tick in enumerate(current_ticks):
|
55
|
+
if tick >= 0:
|
56
|
+
index = _
|
57
|
+
break
|
58
|
+
if decimal is not None:
|
59
|
+
# my_ticks = [(round(float(iii), decimal)) for iii in my_ticks]
|
60
|
+
current_ticks = [f"{val:.{decimal}f}" if val != 0 else "0" for val in current_ticks]
|
61
|
+
|
62
|
+
out_ticks = [f"{val}".replace("-", "\u2212") for val in current_ticks]
|
63
|
+
if add_space:
|
64
|
+
# 在非负数前面加两个空格
|
65
|
+
out_ticks[index:] = [" " + m for m in out_ticks[index:]]
|
66
|
+
|
67
|
+
if ax_x is not None:
|
68
|
+
ax_x.set_xticklabels(out_ticks)
|
69
|
+
return ax_x
|
70
|
+
if ax_y is not None:
|
71
|
+
ax_y.set_yticklabels(out_ticks)
|
72
|
+
return ax_y
|
73
|
+
if cbar is not None:
|
74
|
+
cbar.set_ticklabels(out_ticks)
|
75
|
+
return cbar
|
76
|
+
|
33
77
|
# ** 将生成图片/已有图片制作成动图
|
34
78
|
|
35
79
|
|
@@ -1,24 +1,24 @@
|
|
1
1
|
oafuncs/__init__.py,sha256=2QiNjIIMtstD8y9HWlu23yiZGmmljkNUQknHEbnRwYI,673
|
2
2
|
oafuncs/oa_cmap.py,sha256=LnHI6vMCoFFkMq4P3RgItmJ01Kx5MjjwwlhnaqhRLKI,7242
|
3
3
|
oafuncs/oa_data.py,sha256=H9qZrUziOpc456iIL-1lBwSkBPApl2rlR-ajZg-mDMs,8119
|
4
|
-
oafuncs/oa_draw.py,sha256=
|
4
|
+
oafuncs/oa_draw.py,sha256=R5KONDf3Rp8STXepawtYUTdbcfAK1h6AYh8xiOfac3g,18860
|
5
5
|
oafuncs/oa_file.py,sha256=iHgv0CTH4k_7YUnQ8-qQbLoz_f2lUmVhzGWQ2LkPFP8,11624
|
6
6
|
oafuncs/oa_help.py,sha256=ppNktmtNzs15R20MD1bM7yImlTQ_ngMwvoIglePOKXA,1000
|
7
7
|
oafuncs/oa_nc.py,sha256=7Fp65BJF_PtyaaxS5PS2apA-KkkQLhqh19Xlw__8XMo,12656
|
8
8
|
oafuncs/oa_python.py,sha256=XPTP3o7zTFzfJR_YhsKfQksa3bSYwXsne9YxlJplCEA,3994
|
9
|
-
oafuncs/oa_down/User_Agent-list.txt,sha256=
|
9
|
+
oafuncs/oa_down/User_Agent-list.txt,sha256=pazxSip8_lphEBOPHG902zmIBUg8sBKXgmqp_g6j_E4,661062
|
10
10
|
oafuncs/oa_down/__init__.py,sha256=nY5X7gM1jw7DJxyooR2UJSq4difkw-flz2Ucr_OuDbA,540
|
11
|
-
oafuncs/oa_down/hycom_3hourly.py,sha256=
|
11
|
+
oafuncs/oa_down/hycom_3hourly.py,sha256=pgONwoaLEKPQ4RjoFC4N2b7M6nIvKBgLNwk0XckfyRQ,56030
|
12
12
|
oafuncs/oa_down/literature.py,sha256=dT3-7-beEzQ9mTP8LNV9Gf3q5Z1Pqqjc6FOS010HZeQ,17833
|
13
|
-
oafuncs/oa_down/test.py,sha256=
|
13
|
+
oafuncs/oa_down/test.py,sha256=0IQq3NjqfNr7KkyjS_U-a4mYu-r-E7gzawwo4IfEa6Y,10851
|
14
14
|
oafuncs/oa_sign/__init__.py,sha256=QKqTFrJDFK40C5uvk48GlRRbGFzO40rgkYwu6dYxatM,563
|
15
15
|
oafuncs/oa_sign/meteorological.py,sha256=mLbupsZSq427HTfVbZMvIlFzDHwSzQAbK3X19o8anFY,6525
|
16
16
|
oafuncs/oa_sign/ocean.py,sha256=xrW-rWD7xBWsB5PuCyEwQ1Q_RDKq2KCLz-LOONHgldU,5932
|
17
17
|
oafuncs/oa_sign/scientific.py,sha256=a4JxOBgm9vzNZKpJ_GQIQf7cokkraV5nh23HGbmTYKw,5064
|
18
18
|
oafuncs/oa_tool/__init__.py,sha256=IKOlqpWlb4cMDCtq2VKR_RTxQHDNqR_vfqqsOsp_lKQ,466
|
19
19
|
oafuncs/oa_tool/email.py,sha256=4lJxV_KUzhxgLYfVwYTqp0qxRugD7fvsZkXDe5WkUKo,3052
|
20
|
-
oafuncs-0.0.
|
21
|
-
oafuncs-0.0.
|
22
|
-
oafuncs-0.0.
|
23
|
-
oafuncs-0.0.
|
24
|
-
oafuncs-0.0.
|
20
|
+
oafuncs-0.0.79.dist-info/LICENSE.txt,sha256=rMtLpVg8sKiSlwClfR9w_Dd_5WubTQgoOzE2PDFxzs4,1074
|
21
|
+
oafuncs-0.0.79.dist-info/METADATA,sha256=0fQiVoR8HZQfISZClpFTyPKLXiwnTVA8Iq2Qvzxgj7U,22481
|
22
|
+
oafuncs-0.0.79.dist-info/WHEEL,sha256=pxeNX5JdtCe58PUSYP9upmc7jdRPgvT0Gm9kb1SHlVw,109
|
23
|
+
oafuncs-0.0.79.dist-info/top_level.txt,sha256=bgC35QkXbN4EmPHEveg_xGIZ5i9NNPYWqtJqaKqTPsQ,8
|
24
|
+
oafuncs-0.0.79.dist-info/RECORD,,
|
File without changes
|
File without changes
|
File without changes
|