auto-diffusers 1.3.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- auto_diffusers/__init__.py +6 -0
- auto_diffusers/checker/__init__.py +1 -0
- auto_diffusers/checker/get_checkpoint_type.py +139 -0
- auto_diffusers/checker/memorize_config.py +80 -0
- auto_diffusers/model_path/Perform_path_search.py +228 -0
- auto_diffusers/model_path/__init__.py +7 -0
- auto_diffusers/model_path/create_pipeline.py +253 -0
- auto_diffusers/model_path/flax_config.py +70 -0
- auto_diffusers/model_path/mix_class.py +15 -0
- auto_diffusers/model_path/search_civitai.py +385 -0
- auto_diffusers/model_path/search_hugface.py +418 -0
- auto_diffusers/py.typed +1 -0
- auto_diffusers/setup/Base_config.py +213 -0
- auto_diffusers/setup/__init__.py +1 -0
- auto_diffusers/setup/data_class.py +130 -0
- auto_diffusers/setup/device_config.py +42 -0
- auto_diffusers/setup/runtime_config.py +49 -0
- auto_diffusers/utils/__init__.py +1 -0
- auto_diffusers/utils/device_config.py +40 -0
- auto_diffusers/utils/get_custom_logger.py +22 -0
- auto_diffusers/utils/hide_warning.py +8 -0
- auto_diffusers/utils/process_bar.py +173 -0
- auto_diffusers/utils/run_download.py +140 -0
- auto_diffusers-1.3.1.dist-info/LICENSE +28 -0
- auto_diffusers-1.3.1.dist-info/METADATA +112 -0
- auto_diffusers-1.3.1.dist-info/RECORD +28 -0
- auto_diffusers-1.3.1.dist-info/WHEEL +5 -0
- auto_diffusers-1.3.1.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,130 @@
|
|
|
1
|
+
|
|
2
|
+
from dataclasses import dataclass
|
|
3
|
+
|
|
4
|
+
@dataclass
|
|
5
|
+
class data_config:
|
|
6
|
+
Config_file="model_index.json"
|
|
7
|
+
|
|
8
|
+
VALID_URL_PREFIXES = ["https://huggingface.co/", "huggingface.co/", "hf.co/", "https://hf.co/"]
|
|
9
|
+
exts = [".safetensors", ".ckpt",".bin"]
|
|
10
|
+
|
|
11
|
+
model_dict = {
|
|
12
|
+
"stable diffusion-v2.1" : "stabilityai/stable-diffusion-2-1",
|
|
13
|
+
"waifu diffusion-v1.4": "hakurei/waifu-diffusion",
|
|
14
|
+
"Anything-v3.0": "Linaqruf/anything-v3.0",
|
|
15
|
+
"anything-midjourney-v-4-1": "Joeythemonster/anything-midjourney-v-4-1",
|
|
16
|
+
"Anything-v4.5": "shibal1/anything-v4.5-clone",
|
|
17
|
+
"AB4.5_AC0.2": "aioe/AB4.5_AC0.2",
|
|
18
|
+
"basil_mix": "nuigurumi/basil_mix",
|
|
19
|
+
"Waifu-Diffusers": "Nilaier/Waifu-Diffusers",
|
|
20
|
+
"Double-Exposure-Diffusion": "joachimsallstrom/Double-Exposure-Diffusion",
|
|
21
|
+
"openjourney-v4": "prompthero/openjourney-v4",
|
|
22
|
+
"ACertainThing": "JosephusCheung/ACertainThing",
|
|
23
|
+
"Counterfeit-V2.0": "gsdf/Counterfeit-V2.0",
|
|
24
|
+
"Counterfeit-V2.5": "gsdf/Counterfeit-V2.5",
|
|
25
|
+
"chilled_remix":"chilled_remix",
|
|
26
|
+
"chilled_reversemix":"chilled_reversemix",
|
|
27
|
+
"7th_Layer": "syaimu/7th_test",
|
|
28
|
+
"EimisAnimeDiffusion_1.0v": "eimiss/EimisAnimeDiffusion_1.0v",
|
|
29
|
+
"JWST-Deep-Space-diffusion" : "dallinmackay/JWST-Deep-Space-diffusion",
|
|
30
|
+
"Riga_Collection": "natsusakiyomi/Riga_Collection",
|
|
31
|
+
"sd-db-epic-space-machine" : "rabidgremlin/sd-db-epic-space-machine",
|
|
32
|
+
"spacemidj" : "Falah/spacemidj",
|
|
33
|
+
"anime-kawai-diffusion": "Ojimi/anime-kawai-diffusion",
|
|
34
|
+
"Realistic_Vision_V2.0": "SG161222/Realistic_Vision_V2.0",
|
|
35
|
+
"nasa-space-v2" : "sd-dreambooth-library/nasa-space-v2-768",
|
|
36
|
+
"meinamix_meinaV10": "namvuong96/civit_meinamix_meinaV10",
|
|
37
|
+
"loliDiffusion": "JosefJilek/loliDiffusion",
|
|
38
|
+
}
|
|
39
|
+
exclude = ["safety_checker/model.safetensors",
|
|
40
|
+
"unet/diffusion_pytorch_model.safetensors",
|
|
41
|
+
"vae/diffusion_pytorch_model.safetensors",
|
|
42
|
+
"text_encoder/model.safetensors",
|
|
43
|
+
"unet/diffusion_pytorch_model.fp16.safetensors",
|
|
44
|
+
"text_encoder/model.fp16.safetensors",
|
|
45
|
+
"vae/diffusion_pytorch_model.fp16.safetensors",
|
|
46
|
+
"safety_checker/model.fp16.safetensors",
|
|
47
|
+
|
|
48
|
+
"safety_checker/model.ckpt",
|
|
49
|
+
"unet/diffusion_pytorch_model.ckpt",
|
|
50
|
+
"vae/diffusion_pytorch_model.ckpt",
|
|
51
|
+
"text_encoder/model.ckpt",
|
|
52
|
+
"text_encoder/model.fp16.ckpt",
|
|
53
|
+
"safety_checker/model.fp16.ckpt",
|
|
54
|
+
"unet/diffusion_pytorch_model.fp16.ckpt",
|
|
55
|
+
"vae/diffusion_pytorch_model.fp16.ckpt"]
|
|
56
|
+
|
|
57
|
+
Auto_pipe_class=[
|
|
58
|
+
"AutoPipelineForText2Image",
|
|
59
|
+
"AutoPipelineForImage2Image",
|
|
60
|
+
"AutoPipelineForInpainting",
|
|
61
|
+
]
|
|
62
|
+
|
|
63
|
+
Error_M1 = (
|
|
64
|
+
'''
|
|
65
|
+
Could not load URL.
|
|
66
|
+
Format:"https://huggingface.co/<repo_name>/<model_name>/blob/main/<path_to_file>"
|
|
67
|
+
EX1: "https://huggingface.co/gsdf/Counterfeit-V3.0/blob/main/Counterfeit-V3.0.safetensors"
|
|
68
|
+
EX2: "https://huggingface.co/runwayml/stable-diffusion-v1-5/blob/main/v1-5-pruned.ckpt"
|
|
69
|
+
'''
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
Error_M2= (
|
|
73
|
+
'''
|
|
74
|
+
Could not load hugface_path.
|
|
75
|
+
Format: <repo_name>/<model_name>"
|
|
76
|
+
EX1: "Linaqruf/anything-v3.0"
|
|
77
|
+
EX2: "stabilityai/stable-diffusion-2-1"
|
|
78
|
+
|
|
79
|
+
Suport_model:
|
|
80
|
+
|
|
81
|
+
"stable diffusion-v2.1"
|
|
82
|
+
"waifu diffusion-v1.4"
|
|
83
|
+
"Anything-v3.0"
|
|
84
|
+
"anything-midjourney-v-4-1"
|
|
85
|
+
"Anything-v4.5"
|
|
86
|
+
"AB4.5_AC0.2"
|
|
87
|
+
"basil_mix"
|
|
88
|
+
"Waifu-Diffusers"
|
|
89
|
+
"Double-Exposure-Diffusion"
|
|
90
|
+
"openjourney-v4"
|
|
91
|
+
"ACertainThing"
|
|
92
|
+
"Counterfeit-V2.0"
|
|
93
|
+
"Counterfeit-V2.5"
|
|
94
|
+
"7th_Layer"
|
|
95
|
+
"EimisAnimeDiffusion_1.0v"
|
|
96
|
+
"Riga_Collection"
|
|
97
|
+
"anime-kawai-diffusion"
|
|
98
|
+
"Realistic_Vision_V2.0"
|
|
99
|
+
"meinamix_meinaV10"
|
|
100
|
+
"loliDiffusion"
|
|
101
|
+
''')
|
|
102
|
+
|
|
103
|
+
Error_M3 = ('''
|
|
104
|
+
The specified path could not be recognized. Please try the following
|
|
105
|
+
・Check that the path to the file exists.
|
|
106
|
+
・Check that there is no whitespace in the path.
|
|
107
|
+
・Check if there are any special symbols such as "\" or "." and other special symbols (may not be recognized).
|
|
108
|
+
''')
|
|
109
|
+
config_file_list = [
|
|
110
|
+
"preprocessor_config.json",
|
|
111
|
+
"config.json",
|
|
112
|
+
"model.fp16.safetensors",
|
|
113
|
+
"model.safetensors",
|
|
114
|
+
"pytorch_model.bin",
|
|
115
|
+
"pytorch_model.fp16.bin",
|
|
116
|
+
"scheduler_config.json",
|
|
117
|
+
"merges.txt",
|
|
118
|
+
"special_tokens_map.json",
|
|
119
|
+
"tokenizer_config.json",
|
|
120
|
+
"vocab.json",
|
|
121
|
+
"diffusion_pytorch_model.bin",
|
|
122
|
+
"diffusion_pytorch_model.fp16.bin",
|
|
123
|
+
"diffusion_pytorch_model.fp16.safetensors",
|
|
124
|
+
"diffusion_pytorch_model.non_ema.bin",
|
|
125
|
+
"diffusion_pytorch_model.non_ema.safetensors",
|
|
126
|
+
"diffusion_pytorch_model.safetensors",
|
|
127
|
+
]
|
|
128
|
+
|
|
129
|
+
def __init__(self):
|
|
130
|
+
self.single_file_only = False
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import jax
|
|
2
|
+
|
|
3
|
+
from ..utils.get_custom_logger import custom_logger
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
class device_set:
|
|
7
|
+
def __init__(self):
|
|
8
|
+
self.device_type = self.device_type_check()
|
|
9
|
+
self.logger = custom_logger()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def device_type_check(self):
|
|
13
|
+
_device_type = jax.devices()[0].device_kind
|
|
14
|
+
if "TPU" in _device_type:
|
|
15
|
+
return "TPU"
|
|
16
|
+
elif "cpu" in _device_type:
|
|
17
|
+
return "cpu"
|
|
18
|
+
else:
|
|
19
|
+
return "cuda"
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def extra_device_set(self):
|
|
23
|
+
device_type = self.device_type_check()
|
|
24
|
+
|
|
25
|
+
if device_type == "TPU":
|
|
26
|
+
#import torch_xla.core.xla_model as xm
|
|
27
|
+
#device = xm.xla_device()
|
|
28
|
+
device = device_type
|
|
29
|
+
else:
|
|
30
|
+
device = device_type
|
|
31
|
+
return device
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def count_device(self):
|
|
35
|
+
return jax.device_count()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def is_TPU(self):
|
|
39
|
+
if self.device_type_check() == "TPU":
|
|
40
|
+
return True
|
|
41
|
+
else:
|
|
42
|
+
return False
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import jax
|
|
2
|
+
import re
|
|
3
|
+
import requests
|
|
4
|
+
import importlib
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from .device_config import device_set
|
|
8
|
+
from ..utils.get_custom_logger import custom_logger
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
class Runtime_func(device_set):
|
|
13
|
+
def __init__(self):
|
|
14
|
+
"""
|
|
15
|
+
NOTE:
|
|
16
|
+
Functions that should be included in the `Basic_config` class are needed in the `install_packages` class, so these functions are separated into the `runtime_func` class.
|
|
17
|
+
This is because the `install_packages` class is executed before the `Basic_config` class is executed, so these functions are separated and inherited later.
|
|
18
|
+
"""
|
|
19
|
+
self.device_type = self.device_type_check()
|
|
20
|
+
self.use_TPU = self.is_TPU()
|
|
21
|
+
self.logger = custom_logger()
|
|
22
|
+
super().__init__()
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def module_version(self,module_name):
|
|
26
|
+
try:
|
|
27
|
+
version = importlib.metadata.version(module_name)
|
|
28
|
+
return re.match(r"^\d+\.\d+\.\d+", version).group(0)
|
|
29
|
+
except importlib.metadata.PackageNotFoundError:
|
|
30
|
+
return None
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def is_url_valid(self,url) -> bool:
|
|
34
|
+
response = requests.head(url)
|
|
35
|
+
try:
|
|
36
|
+
response.raise_for_status()
|
|
37
|
+
except requests.RequestException:
|
|
38
|
+
return False
|
|
39
|
+
else:
|
|
40
|
+
return True
|
|
41
|
+
finally:
|
|
42
|
+
self.logger.debug(f"response.status_code: {response.status_code}")
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def DEBUG(self,debug:bool):
|
|
46
|
+
if debug:
|
|
47
|
+
self.logger.setLevel(logging.DEBUG)
|
|
48
|
+
else:
|
|
49
|
+
self.logger.setLevel(logging.WARNING)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
from .run_download import download
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import jax
|
|
2
|
+
from get_custom_logger import logger
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
class device_set:
|
|
6
|
+
def __init__(self):
|
|
7
|
+
self.device_type = self.device_type_check()
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def device_type_check(self):
|
|
11
|
+
_device_type = jax.devices()[0].device_kind
|
|
12
|
+
if "TPU" in _device_type:
|
|
13
|
+
return "TPU"
|
|
14
|
+
elif "cpu" in _device_type:
|
|
15
|
+
return "cpu"
|
|
16
|
+
else:
|
|
17
|
+
return "cuda"
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def device_set(self):
|
|
21
|
+
device_type = self.device_type_check()
|
|
22
|
+
logger.debug(f"device_type: {self.device_type}")
|
|
23
|
+
if device_type == "TPU":
|
|
24
|
+
#import torch_xla.core.xla_model as xm
|
|
25
|
+
#device = xm.xla_device()
|
|
26
|
+
device = device_type
|
|
27
|
+
else:
|
|
28
|
+
device = device_type
|
|
29
|
+
return device
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def count_device(self):
|
|
33
|
+
return jax.device_count()
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def is_TPU(self):
|
|
37
|
+
if self.device_type_check() == "TPU":
|
|
38
|
+
return True
|
|
39
|
+
else:
|
|
40
|
+
return False
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import logging
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
def custom_logger(debug=None):
|
|
6
|
+
_debugs = os.environ.get("DEBUG",False) if debug is None else debug
|
|
7
|
+
format = '%(levelname)s:<cell line: %(lineno)d> <funcName: %(funcName)s>: %(message)s'
|
|
8
|
+
logger = logging.getLogger(__name__)
|
|
9
|
+
logger.propagate = False
|
|
10
|
+
if not logger.handlers:
|
|
11
|
+
formatter = logging.Formatter(format)
|
|
12
|
+
handler = logging.StreamHandler()
|
|
13
|
+
handler.setFormatter(formatter)
|
|
14
|
+
logger.addHandler(handler)
|
|
15
|
+
if _debugs:
|
|
16
|
+
logger.setLevel(logging.DEBUG)
|
|
17
|
+
else:
|
|
18
|
+
logger.setLevel(logging.WARNING)
|
|
19
|
+
return logger
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
logger = custom_logger()
|
|
@@ -0,0 +1,173 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import time
|
|
3
|
+
import queue
|
|
4
|
+
import threading
|
|
5
|
+
from tqdm.auto import tqdm
|
|
6
|
+
|
|
7
|
+
class ProcessBarRun:
|
|
8
|
+
"""
|
|
9
|
+
Example:
|
|
10
|
+
if __name__ == "__main__":
|
|
11
|
+
bar = ProcessBarRun(total=11)
|
|
12
|
+
for s in range(4):
|
|
13
|
+
bar.bar_update(3)
|
|
14
|
+
time.sleep(1)
|
|
15
|
+
|
|
16
|
+
bar.bar_update(exit=True)
|
|
17
|
+
|
|
18
|
+
Args:
|
|
19
|
+
desc:str = "Running",
|
|
20
|
+
default_desc:str = "Running",
|
|
21
|
+
fin_desc = "",
|
|
22
|
+
pofix:str="",
|
|
23
|
+
default_pofix:str = "",
|
|
24
|
+
fin_pofix= "Finish!",
|
|
25
|
+
desc_dot:bool = False,
|
|
26
|
+
pofix_dot:bool = False
|
|
27
|
+
|
|
28
|
+
If the download_bar is true, the URL and save_path are required
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
def __init__(self,
|
|
32
|
+
total: int = 0,
|
|
33
|
+
desc: str = "",
|
|
34
|
+
default_desc: str = "",
|
|
35
|
+
fin_desc="",
|
|
36
|
+
pofix: str = "",
|
|
37
|
+
default_pofix: str = "",
|
|
38
|
+
fin_pofix="Finish!",
|
|
39
|
+
desc_dot: bool = False,
|
|
40
|
+
pofix_dot: bool = False,
|
|
41
|
+
download_bar:bool = False,
|
|
42
|
+
url:str = "",
|
|
43
|
+
save_path:str = "",
|
|
44
|
+
**ex_word):
|
|
45
|
+
|
|
46
|
+
self.total = total
|
|
47
|
+
self.default_desc = default_desc
|
|
48
|
+
self.default_pofix = default_pofix
|
|
49
|
+
self.fin_desc = fin_desc
|
|
50
|
+
self.fin_pofix = fin_pofix
|
|
51
|
+
self.desc_dot = desc_dot
|
|
52
|
+
self.pofix_dot = pofix_dot
|
|
53
|
+
self.queue_obj = queue.Queue()
|
|
54
|
+
self._count = 0
|
|
55
|
+
self._run_count = 0
|
|
56
|
+
self.dot_count = 0
|
|
57
|
+
self.max_dots = 5
|
|
58
|
+
self.desc = desc
|
|
59
|
+
self.pofix = pofix
|
|
60
|
+
self.base_desc_txt = ""
|
|
61
|
+
self.base_pofix_txt = ""
|
|
62
|
+
self.stop_event = threading.Event()
|
|
63
|
+
self.stop_dot = threading.Event()
|
|
64
|
+
self.tqdm_lock = threading.Lock()
|
|
65
|
+
self.tqdm_obj = tqdm(total=total, desc=desc, postfix=pofix)
|
|
66
|
+
self.arg_update(desc=desc, postfix=pofix, **ex_word)
|
|
67
|
+
self.dot_thread = threading.Thread(target=self.prosess_dot)
|
|
68
|
+
self.dot_thread.start()
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def __del__(self):
|
|
73
|
+
self.stop()
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def __enter__(self):
|
|
77
|
+
return self
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
81
|
+
self.stop()
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def stop(self):
|
|
85
|
+
self.stop_event.set()
|
|
86
|
+
self.stop_dot.set()
|
|
87
|
+
self.tqdm_obj.n = self.total
|
|
88
|
+
if self.fin_desc:
|
|
89
|
+
setattr(self.tqdm_obj,"desc",self.fin_desc)
|
|
90
|
+
if self.fin_pofix:
|
|
91
|
+
setattr(self.tqdm_obj,"postfix",self.fin_pofix)
|
|
92
|
+
self.tqdm_obj.refresh()
|
|
93
|
+
self.tqdm_obj.close()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def arg_update(self, *ex_word, **input_dict):
|
|
97
|
+
for extra_word in ex_word:
|
|
98
|
+
if isinstance(extra_word, dict):
|
|
99
|
+
for _key, _value in extra_word.items():
|
|
100
|
+
setattr(self, _key, _value)
|
|
101
|
+
for key, value in input_dict.items():
|
|
102
|
+
if hasattr(self, key):
|
|
103
|
+
setattr(self, key, value)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def run_downlaod_with_bar(self,response):
|
|
110
|
+
with self.tqdm_lock:
|
|
111
|
+
for chunk in response.iter_content(chunk_size=4096):
|
|
112
|
+
self.tqdm_obj.write(chunk)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def prosess_dot(self):
|
|
116
|
+
"""
|
|
117
|
+
NOTE:
|
|
118
|
+
set_description_str and set_description_str are not used,
|
|
119
|
+
because they cannot be used in the case of the download_with_bar function.
|
|
120
|
+
"""
|
|
121
|
+
self.dot_count = 0
|
|
122
|
+
chenge_check = False
|
|
123
|
+
while not self.stop_dot.is_set():
|
|
124
|
+
for num in range(self.max_dots):
|
|
125
|
+
with self.tqdm_lock:
|
|
126
|
+
dot_txt = "." * num
|
|
127
|
+
if self.desc and self.desc_dot:
|
|
128
|
+
desc_dot_txt = dot_txt
|
|
129
|
+
else:
|
|
130
|
+
desc_dot_txt = ""
|
|
131
|
+
|
|
132
|
+
if self.pofix and self.pofix_dot:
|
|
133
|
+
pofix_dot_txt = dot_txt
|
|
134
|
+
else:
|
|
135
|
+
pofix_dot_txt = ""
|
|
136
|
+
|
|
137
|
+
setattr(self.tqdm_obj, "desc", self.desc + desc_dot_txt)
|
|
138
|
+
setattr(self.tqdm_obj, "postfix", self.pofix + pofix_dot_txt)
|
|
139
|
+
self.tqdm_obj.refresh()
|
|
140
|
+
|
|
141
|
+
if self.stop_dot.is_set():
|
|
142
|
+
break
|
|
143
|
+
time.sleep(0.5)
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def bar_update(
|
|
147
|
+
self,
|
|
148
|
+
update_rate: int = 1,
|
|
149
|
+
exit: bool = False,
|
|
150
|
+
desc = None,
|
|
151
|
+
pofix = None,
|
|
152
|
+
desc_dot = None,
|
|
153
|
+
pofix_dot= None):
|
|
154
|
+
"""
|
|
155
|
+
args:
|
|
156
|
+
desc : str
|
|
157
|
+
pofix : str
|
|
158
|
+
desc_dot : bool
|
|
159
|
+
pofix_dot : bool
|
|
160
|
+
"""
|
|
161
|
+
self.desc = desc or self.desc or self.default_desc
|
|
162
|
+
self.pofix = pofix or self.pofix or self.default_pofix
|
|
163
|
+
self.desc_dot = desc_dot or self.desc_dot
|
|
164
|
+
self.pofix_dot = pofix_dot or self.pofix_dot
|
|
165
|
+
self.tqdm_obj.update(int(update_rate))
|
|
166
|
+
self.tqdm_obj.set_description_str(self.desc)
|
|
167
|
+
self.tqdm_obj.set_postfix_str(self.pofix)
|
|
168
|
+
self.tqdm_obj.refresh()
|
|
169
|
+
if exit:
|
|
170
|
+
self.stop()
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import os
|
|
2
|
+
import requests
|
|
3
|
+
import time
|
|
4
|
+
import queue
|
|
5
|
+
import threading
|
|
6
|
+
from tqdm.auto import tqdm
|
|
7
|
+
|
|
8
|
+
#from ..setup.Base_config import Basic_config
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class download:
|
|
12
|
+
"""
|
|
13
|
+
Example:
|
|
14
|
+
if __name__ == "__main__":
|
|
15
|
+
file_path = download(url=<url>,save_path=<save_path>)
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
desc:str = "Running",
|
|
19
|
+
fin_desc = "",
|
|
20
|
+
pofix:str="",
|
|
21
|
+
fin_pofix= "Finish!",
|
|
22
|
+
desc_dot:bool = False,
|
|
23
|
+
pofix_dot:bool = False
|
|
24
|
+
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
def __init__(self,
|
|
28
|
+
url: str,
|
|
29
|
+
save_path: str,
|
|
30
|
+
desc: str = "",
|
|
31
|
+
fin_desc = "",
|
|
32
|
+
pofix: str = "",
|
|
33
|
+
fin_pofix = "Finish!",
|
|
34
|
+
desc_dot: bool = False,
|
|
35
|
+
pofix_dot: bool = False,
|
|
36
|
+
**ex_word):
|
|
37
|
+
super().__init__()
|
|
38
|
+
self.url = url
|
|
39
|
+
self.save_path = save_path
|
|
40
|
+
self.desc = desc
|
|
41
|
+
self.pofix = pofix
|
|
42
|
+
self.fin_desc = fin_desc
|
|
43
|
+
self.fin_pofix = fin_pofix
|
|
44
|
+
self.desc_dot = desc_dot
|
|
45
|
+
self.pofix_dot = pofix_dot
|
|
46
|
+
self.queue_obj = queue.Queue()
|
|
47
|
+
self._count = 0
|
|
48
|
+
self._run_count = 0
|
|
49
|
+
self.dot_count = 0
|
|
50
|
+
self.max_dots = 5
|
|
51
|
+
self.desc = desc
|
|
52
|
+
self.pofix = pofix
|
|
53
|
+
self.base_desc_txt = ""
|
|
54
|
+
self.base_pofix_txt = ""
|
|
55
|
+
self.stop_event = threading.Event()
|
|
56
|
+
self.stop_dot = threading.Event()
|
|
57
|
+
self.tqdm_lock = threading.Lock()
|
|
58
|
+
self.response = self.response_get(url)
|
|
59
|
+
os.makedirs(os.path.dirname(save_path),exist_ok=True)
|
|
60
|
+
self.tqdm_obj = tqdm.wrapattr(
|
|
61
|
+
open(save_path, "wb"),
|
|
62
|
+
"write",
|
|
63
|
+
miniters=1,
|
|
64
|
+
desc=desc,
|
|
65
|
+
postfix=pofix,
|
|
66
|
+
total=int(self.response.headers.get('content-length', 0)),
|
|
67
|
+
unit_scale=True,
|
|
68
|
+
**ex_word
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
self.dot_thread.start()
|
|
72
|
+
with self.tqdm_lock:
|
|
73
|
+
for chunk in self.response.iter_content(chunk_size=4096):
|
|
74
|
+
self.tqdm_obj.write(chunk)
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def __del__(self):
|
|
78
|
+
self.stop()
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def __enter__(self):
|
|
82
|
+
return self
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def __exit__(self, exc_type, exc_val, exc_tb):
|
|
86
|
+
self.stop()
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def stop(self):
|
|
90
|
+
self.stop_event.set()
|
|
91
|
+
self.stop_dot.set()
|
|
92
|
+
self.tqdm_obj.n = self.total
|
|
93
|
+
if self.fin_desc:
|
|
94
|
+
setattr(self.tqdm_obj,"desc",self.fin_desc)
|
|
95
|
+
if self.fin_pofix:
|
|
96
|
+
setattr(self.tqdm_obj,"postfix",self.fin_pofix)
|
|
97
|
+
self.tqdm_obj.refresh()
|
|
98
|
+
self.tqdm_obj.close()
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def response_get(self,url):
|
|
102
|
+
if not self.is_url_valid(url):
|
|
103
|
+
raise requests.HTTPError("URL is invalid.")
|
|
104
|
+
response = requests.get(url, stream=True)
|
|
105
|
+
try:
|
|
106
|
+
response.raise_for_status()
|
|
107
|
+
except requests.HTTPError:
|
|
108
|
+
raise requests.HTTPError(f"Invalid URL: {response.status_code}")
|
|
109
|
+
|
|
110
|
+
return response
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def prosess_dot(self):
|
|
114
|
+
"""
|
|
115
|
+
NOTE:
|
|
116
|
+
set_description_str and set_description_str are not used,
|
|
117
|
+
because they cannot be used in the case of the download_with_bar function.
|
|
118
|
+
"""
|
|
119
|
+
self.dot_count = 0
|
|
120
|
+
while not self.stop_dot.is_set():
|
|
121
|
+
for num in range(self.max_dots):
|
|
122
|
+
with self.tqdm_lock:
|
|
123
|
+
dot_txt = "." * num
|
|
124
|
+
if self.desc and self.desc_dot:
|
|
125
|
+
desc_dot_txt = dot_txt
|
|
126
|
+
else:
|
|
127
|
+
desc_dot_txt = ""
|
|
128
|
+
|
|
129
|
+
if self.pofix and self.pofix_dot:
|
|
130
|
+
pofix_dot_txt = dot_txt
|
|
131
|
+
else:
|
|
132
|
+
pofix_dot_txt = ""
|
|
133
|
+
|
|
134
|
+
setattr(self.tqdm_obj, "desc", self.desc + desc_dot_txt)
|
|
135
|
+
setattr(self.tqdm_obj, "postfix", self.pofix + pofix_dot_txt)
|
|
136
|
+
self.tqdm_obj.refresh()
|
|
137
|
+
|
|
138
|
+
if self.stop_dot.is_set():
|
|
139
|
+
break
|
|
140
|
+
time.sleep(0.5)
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
BSD 3-Clause License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2024, suzukimain
|
|
4
|
+
|
|
5
|
+
Redistribution and use in source and binary forms, with or without
|
|
6
|
+
modification, are permitted provided that the following conditions are met:
|
|
7
|
+
|
|
8
|
+
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
+
list of conditions and the following disclaimer.
|
|
10
|
+
|
|
11
|
+
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
+
this list of conditions and the following disclaimer in the documentation
|
|
13
|
+
and/or other materials provided with the distribution.
|
|
14
|
+
|
|
15
|
+
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
+
contributors may be used to endorse or promote products derived from
|
|
17
|
+
this software without specific prior written permission.
|
|
18
|
+
|
|
19
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
+
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
+
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
+
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
+
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
+
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
+
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
+
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
+
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|