auto-diffusers 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,418 @@
1
+ import re
2
+ import requests
3
+ import gc
4
+ from requests import HTTPError
5
+ from tqdm.auto import tqdm
6
+
7
+ from diffusers import (DiffusionPipeline,
8
+ StableDiffusionPipeline,
9
+ FlaxDiffusionPipeline)
10
+
11
+ from huggingface_hub import hf_hub_download
12
+
13
+ from ..setup.Base_config import Basic_config
14
+
15
+
16
+ class Huggingface(Basic_config):
17
+ def __init__(self):
18
+ super().__init__()
19
+ self.num_prints=20
20
+ self.model_id=""
21
+ self.model_name=""
22
+ self.vae_name=""
23
+ self.model_file=""
24
+ self.input_url=False
25
+ self.diffuser_model=False
26
+ self.check_choice_key = ""
27
+ self.choice_number = -1
28
+ self.file_path_dict={}
29
+ self.special_file=""
30
+ self.hf_repo_id = ""
31
+ #self.model_select = ""
32
+
33
+
34
+ def repo_name_or_path(self,model_name_or_path):
35
+ pattern = r"([^/]+)/([^/]+)/(?:blob/main/)?(.+)"
36
+ weights_name = None
37
+ repo_id = None
38
+ for prefix in self.VALID_URL_PREFIXES:
39
+ model_name_or_path = model_name_or_path.replace(prefix, "")
40
+ match = re.match(pattern, model_name_or_path)
41
+ if not match:
42
+ return repo_id, weights_name
43
+ repo_id = f"{match.group(1)}/{match.group(2)}"
44
+ weights_name = match.group(3)
45
+ return repo_id, weights_name
46
+
47
+
48
+ def run_hf_download(self,url_or_path):
49
+ """
50
+ retrun:
51
+ [os.path(str), Single_file_or_not(bool)]
52
+ """
53
+ def _hf_repo_download(path):
54
+ model_path = DiffusionPipeline.download(path)
55
+ return model_path
56
+
57
+ single_file = True
58
+
59
+ if any(url_or_path.startswith(checked) for checked in self.VALID_URL_PREFIXES):
60
+ if not self.is_url_valid(url_or_path):
61
+ raise HTTPError("Invalid URL")
62
+ hf_path, file_name =self.repo_name_or_path(url_or_path)
63
+ self.logger.debug(f"url_or_path:{url_or_path}")
64
+ self.logger.debug(f"hf_path: {hf_path} \nfile_name: {file_name}")
65
+ if hf_path and file_name:
66
+ single_file = True
67
+ model_file_path = hf_hub_download(hf_path, file_name)
68
+ elif hf_path and (not file_name):
69
+ if self.diffusers_model_check(hf_path):
70
+ single_file = False
71
+ model_file_path = _hf_repo_download(url_or_path)
72
+ else:
73
+ raise HTTPError("Invalid hf_path")
74
+ else:
75
+ raise TypeError("Invalid path_or_url")
76
+
77
+ #from hf_repo
78
+ elif self.diffusers_model_check(url_or_path):
79
+ self.logger.debug(f"url_or_path: {url_or_path}")
80
+ single_file = False
81
+ model_file_path = _hf_repo_download(url_or_path)
82
+ else:
83
+ self.logger.debug(f"url_or_path:{url_or_path}")
84
+ raise TypeError("Invalid path_or_url")
85
+ return [model_file_path, single_file]
86
+
87
+
88
+ def model_safe_check(self,model_list) ->str:
89
+ if len(model_list)>1:
90
+ for check_model in model_list:
91
+ match = bool(re.search(r"(?i)[-_]sfw", check_model))
92
+ if match:
93
+ return check_model
94
+ return model_list[0]
95
+
96
+
97
+ def list_safe_check(self,model_list) -> list:
98
+ for check_model in model_list:
99
+ if bool(re.search(r"(?i)[-ー__]sfw", check_model)):
100
+ model_list.remove(check_model)
101
+ model_list.insert(0, check_model)
102
+ break
103
+ return model_list
104
+
105
+
106
+ def diffusers_model_check(self,checked_model: str) -> bool:
107
+ index_url=f"https://huggingface.co/{checked_model}/blob/main/model_index.json"
108
+ return self.is_url_valid(index_url)
109
+
110
+
111
+ def hf_model_check(self,path) -> bool:
112
+ return self.is_url_valid(f"https://huggingface.co/{path}")
113
+
114
+
115
+ def data_get(self,path) -> list:
116
+ url = f"https://huggingface.co/api/models/{path}"
117
+ data = requests.get(url).json()
118
+ file_value_list = []
119
+ df_model_bool=False
120
+ #fix error': 'Repo model <repo_id>/<model> is gated. You must be authenticated to access it.
121
+ try:
122
+ siblings=data["siblings"]
123
+ except KeyError:
124
+ return []
125
+
126
+ for item in siblings:
127
+ data["siblings"]
128
+ file_path=item["rfilename"]
129
+ #model_index.json outside the root directory is not recognized
130
+ if file_path=="model_index.json" and (not self.single_file_only):
131
+ df_model_bool=True
132
+ elif (
133
+ any(file_path.endswith(ext) for ext in self.exts) and
134
+ not any(file_path.endswith(ex) for ex in self.exclude)
135
+ ):
136
+ file_value_list.append(file_path)
137
+ #↓{df_model,file_value_list}
138
+ self.file_path_dict.update({path:(df_model_bool,file_value_list)})
139
+ return file_value_list
140
+
141
+
142
+ def hf_model_search(self,
143
+ model_path,
144
+ limit_num):
145
+ url = f"https://huggingface.co/api/models"#?search={model_name}"
146
+ params={"search":model_path,"sort":"likes","direction":-1,"limit":limit_num}#"downloads",}
147
+ return requests.get(url,params=params).json()
148
+
149
+
150
+ def hf_models(self,
151
+ model_name,
152
+ limit):
153
+ """
154
+ return:
155
+ repo_model_list,with_like : list
156
+ """
157
+ #self.logger.debug(f"model_name: {model_name}")
158
+ data=self.hf_model_search(model_name,limit)
159
+ final_list = []
160
+ if data:
161
+ for item in data:
162
+ model_id,like,private_value,tag_value = item["modelId"],item["likes"],item["private"],item["tags"]
163
+ if ("audio-to-audio" not in tag_value and
164
+ (not private_value)):
165
+ if self.data_get(model_id):
166
+ model_dict = {"model_id":model_id,
167
+ "like":like,}
168
+ final_list.append(model_dict)
169
+ else:
170
+ print("No models matching your criteria were found on huggingface.")
171
+ return []
172
+ return final_list
173
+
174
+
175
+ def model_name_search(self,
176
+ model_name: str,
177
+ auto_set: bool,
178
+ Recursive_execution:bool = False):
179
+
180
+ def find_max_like(model_dict_list:list):
181
+ """
182
+ Finds the dictionary with the highest "like" value in a list of dictionaries.
183
+
184
+ Args:
185
+ model_dict_list: A list of dictionaries.
186
+
187
+ Returns:
188
+ The dictionary with the highest "like" value, or the first dictionary if none have "like".
189
+ """
190
+ max_like = 0
191
+ max_like_dict = None
192
+ for model_dict in model_dict_list:
193
+ if model_dict["like"] > max_like:
194
+ max_like = model_dict["like"]
195
+ max_like_dict = model_dict
196
+ return max_like_dict["model_id"] or model_dict_list[0]["model_id"]
197
+
198
+ """
199
+ auto_set: bool
200
+ loads the model with the most likes in hugface
201
+ """
202
+ if Recursive_execution:
203
+ limit = 1000
204
+ else:
205
+ limit = 15
206
+
207
+ repo_model_list = self.hf_models(model_name,limit)
208
+ model_history = self.check_func_hist(key="hf_model_name",
209
+ return_value=True)
210
+ if not auto_set:
211
+ print("\033[34mThe following model paths were found")
212
+ if model_history is not None:
213
+ print(f"Previous Choice: {model_history}")
214
+ print("0.Search civitai")
215
+ for (i,(model_dict)) in enumerate(repo_model_list,1):
216
+ model_name = model_dict["model_id"]
217
+ like = model_dict["like"]
218
+ print(f"{i}.model path: {model_name}, evaluation: {like}")
219
+
220
+ if Recursive_execution:
221
+ print("16.Other than above")
222
+
223
+ while True:
224
+ try:
225
+ choice = int(input("Select the model path to use: "))
226
+ except ValueError:
227
+ print("\033[33mOnly natural numbers are valid.\033[34m")
228
+ continue
229
+ if choice == 0:
230
+ return "_hf_no_model"
231
+ elif (not Recursive_execution) and choice>=16 and choice == len(repo_model_list)+1:
232
+ return self.model_name_search(model_name = model_name,
233
+ auto_set = auto_set,
234
+ Recursive_execution = True)
235
+ elif 1 <= choice <= len(repo_model_list):
236
+ choice_path_dict = repo_model_list[choice-1]
237
+ choice_path = choice_path_dict["model_id"]
238
+ break
239
+ else:
240
+ print(f"Please enter the numbers 1~{len(repo_model_list)}")
241
+
242
+ else:
243
+ if repo_model_list:
244
+ choice_path = find_max_like(repo_model_list)
245
+ else:
246
+ choice_path = "_hf_no_model"
247
+
248
+ return choice_path
249
+
250
+
251
+ def file_name_set_sub(self,model_select,file_value,model_type):
252
+ check_key = f"{model_select}_select"
253
+ if not file_value and (not self.diffuser_model):
254
+ print("\033[31mNo candidates found at huggingface\033[0m")
255
+ res = input("Searching for civitai?: ")
256
+ if res.lower() in ["y","yes"]:
257
+ return "_hf_no_model"
258
+ else:
259
+ raise ValueError("No available files were found in the specified repository")
260
+ elif not file_value:
261
+ print("\033[34mOnly models in Diffusers format found")
262
+ while True:
263
+ result=input("Do you want to use it?[y/n]: ")
264
+ if result.lower() in ["y","yes"]:
265
+ return "_DFmodel"
266
+ elif result.lower() in ["n","no"]:
267
+ sec_result=input("Searching for civitai?[y/n]: ")
268
+ if sec_result.lower() in ["y","yes"]:
269
+ return "_hf_no_model"
270
+ elif sec_result.lower() in ["n","no"]:
271
+ raise ValueError("Processing was stopped because no corresponding model was found.")
272
+ else:
273
+ print("Please enter only [y,n]")
274
+
275
+ file_value=self.list_safe_check(file_value)
276
+ if len(file_value)>=self.num_prints: #15
277
+ start_number="1"
278
+ #previous_select = self.check_func_hist(key=check_key)
279
+ #if previous_select:
280
+ choice_history = self.check_func_hist(key = check_key,return_value=True)
281
+ if choice_history:
282
+ if choice_history>self.num_prints+1:
283
+ choice_history = self.num_prints+1
284
+ print(f"\033[33m*Previous number: {choice_history}\033[0m")
285
+
286
+ if self.diffuser_model:
287
+ start_number="0"
288
+ print("\033[34m0.Use Diffusers format model")
289
+ for i in range(self.num_prints):
290
+ print(f"\033[34m{i+1}.File name: {file_value[i]}\033[0m")
291
+ print(f"\033[34m{self.num_prints+1}.Other than the files listed above (all candidates will be displayed)\n")
292
+ while True:
293
+ choice = input(f"select the file you want to use({start_number}~21): ")
294
+ try:
295
+ choice=int(choice)
296
+ except ValueError:
297
+ print("\033[33mOnly natural numbers are valid\033[34m")
298
+ continue
299
+ if self.diffuser_model and choice==0:
300
+ old_num=None
301
+ self.input_url=False
302
+ self.choice_number = -1
303
+ print("\033[0m",end="")
304
+ choice_history_update = self.check_func_hist(key=check_key,value=choice,update=True)
305
+ return "_DFmodel"
306
+
307
+ elif choice==(self.num_prints+1): #other_file
308
+ break
309
+ elif 1<=choice<=self.num_prints:
310
+ self.input_url=True
311
+ old_num=choice
312
+ choice_path=file_value[choice-1]
313
+ self.choice_number = choice
314
+ print("\033[0m",end="")
315
+ choice_history_update = self.check_func_hist(key=check_key,value=choice,update=True)
316
+ return choice_path
317
+ else:
318
+ print(f"\033[33mPlease enter numbers from 1~{self.num_prints}\033[34m")
319
+ print("\033[0m",end="")
320
+ print("\n\n")
321
+
322
+ choice_history = self.check_func_hist(key = check_key,return_value=True)
323
+ if choice_history:
324
+ print(f"\033[33m*Previous number: {choice_history}\033[0m")
325
+
326
+ start_number="1"
327
+ if self.diffuser_model:
328
+ start_number="0"
329
+ print("\033[34m0.Use Diffusers format model\033[0m")
330
+ for i, file_name in enumerate(file_value, 1):
331
+ print(f"\033[34m{i}.File name: {file_name}")
332
+ while True:
333
+ choice = input(f"Select the file you want to use({start_number}~{len(file_value)}): ")
334
+ try:
335
+ choice=int(choice)
336
+ except ValueError:
337
+ print("\033[33mOnly natural numbers are valid\033[34m")
338
+ else:
339
+ if self.diffuser_model and choice==0:
340
+ self.input_url=False
341
+ print("\033[0m",end="")
342
+ self.choice_number = -1
343
+ choice_history_update = self.check_func_hist(key=check_key,value=choice,update=True)
344
+ return "_DFmodel"
345
+ if 1<=choice<=len(file_value):
346
+ self.input_url=True
347
+ old_num=choice
348
+ choice_path=file_value[choice-1]
349
+ self.choice_number = choice
350
+ print("\033[0m",end="")
351
+ choice_history_update = self.check_func_hist(key=check_key,value=choice,update=True)
352
+ return choice_path
353
+ else:
354
+ print(f"\033[33mPlease enter numbers from 1~{len(file_value)}\033[34m")
355
+ #print("\033[0m",end="")
356
+
357
+
358
+ def file_name_set(
359
+ self,
360
+ model_select,
361
+ auto,
362
+ model_type="Checkpoint",
363
+ download=False,
364
+ single_file_only=False
365
+ ):
366
+
367
+ if self.diffusers_model_check(model_select) and model_type=="Checkpoint":
368
+ self.diffuser_model=True
369
+ #check_choice_key = f"model_select_{model_type}"
370
+ url = f"https://huggingface.co/api/models/{model_select}"
371
+ try:
372
+ response = requests.get(url)
373
+ response.raise_for_status()
374
+ except requests.exceptions.HTTPError:
375
+ raise HTTPError("A hugface login or token is required")
376
+ data = response.json()
377
+ choice_path=""
378
+ file_value = []
379
+ siblings = data["siblings"]
380
+ if data:
381
+ for item in siblings:
382
+ fi_path=item["rfilename"]
383
+ if (any(fi_path.endswith(ext) for ext in self.exts) and
384
+ (not any(fi_path.endswith(ex) for ex in self.exclude)) and
385
+ (not any(fi_path.endswith(st) for st in self.config_file_list))):
386
+ file_value.append(fi_path)
387
+ else:
388
+ raise ValueError("No available file was found.\nPlease check the name.")
389
+ if file_value:
390
+ file_value=self.sort_by_version(file_value)
391
+ if not auto:
392
+ print("\033[34mThe following model files were found\033[0m")
393
+ choice_path=self.file_name_set_sub(model_select,file_value,model_type)
394
+ #if not self.choice_number == -1:
395
+ # choice_key_update = self.check_func_hist(key=check_key,value=self.choice_number)
396
+ else:
397
+ if self.diffuser_model and (not self.single_file_only):
398
+ self.input_url=False
399
+ choice_path="_DFmodel"
400
+
401
+ else:
402
+ self.input_url=True
403
+ choice_path=self.model_safe_check(file_value)
404
+
405
+
406
+ elif self.diffuser_model:
407
+ if not auto:
408
+ self.logger.warning("\033[32mOnly models in Diffusers format found\033[0m")
409
+ choice_path = "_DFmodel"
410
+ else:
411
+ raise FileNotFoundError("No available files found in the specified repository")
412
+ #if model_type!="Checkpoint" and model_type!="_DFmodel":
413
+ #self.input_url=False
414
+ if download and not choice_path=="_DFmodel":
415
+ choice_path=hf_hub_download(repo_id=model_select, filename=choice_path)
416
+ #if not self.choice_number== -1:
417
+ # choice_key_update = self.check_func_hist(key=check_choice_key,value=self.choice_number)
418
+ return choice_path
@@ -0,0 +1 @@
1
+
@@ -0,0 +1,213 @@
1
+ import os
2
+ import jax
3
+ import inspect
4
+ import difflib
5
+
6
+ import diffusers
7
+ from natsort import natsorted
8
+
9
+ from .device_config import device_set
10
+ from .data_class import data_config
11
+ from ..checker.memorize_config import config_check
12
+ from .runtime_config import Runtime_func
13
+ from ..utils.get_custom_logger import custom_logger
14
+
15
+ class Basic_config(
16
+ data_config,
17
+ config_check,
18
+ Runtime_func,
19
+ device_set
20
+ ):
21
+
22
+ def __init__(self):
23
+ super().__init__()
24
+ self.device_count = self.count_device()
25
+ self.device_type = self.device_type_check()
26
+ self.device = self.extra_device_set()
27
+ self.use_TPU = self.is_TPU()
28
+ self.logger = custom_logger()
29
+
30
+
31
+ @classmethod
32
+ def get_inherited_class(cls,class_name) -> list:
33
+ inherited_class = inspect.getmro(class_name)
34
+ return [cls_method.__name__ for cls_method in inherited_class]
35
+
36
+
37
+ def get_item(self,dict_obj):
38
+ """
39
+ Returns the first element of the dictionary
40
+ """
41
+ return next(iter(dict_obj.items()))[1]
42
+
43
+
44
+ def pipe_class_type(
45
+ self,
46
+ class_name
47
+ ):
48
+ """
49
+ Args:
50
+ class_name : class
51
+
52
+ Returns:
53
+ Literal['txt2img','img2img','txt2video']
54
+ """
55
+ _txt2img_method_list = [] #else
56
+ _img2img_method_list = ["image"]
57
+ _img2video_method_list = ["video_length","fps"]
58
+
59
+ call_method = self.get_call_method(class_name,method_name = '__call__')
60
+
61
+ if any(method in call_method for method in _img2video_method_list):
62
+ pipeline_type = "txt2video"
63
+ elif any(method in call_method for method in _img2img_method_list):
64
+ pipeline_type = "img2img"
65
+ else:
66
+ pipeline_type = "txt2img"
67
+ return pipeline_type
68
+
69
+
70
+ def pipeline_metod_type(self,Target_class) -> str:
71
+ """
72
+ Args:
73
+ Target_class : class
74
+
75
+ Returns:
76
+ Literal['torch','flax','onnx']
77
+ """
78
+ torch_list=["DiffusionPipeline",
79
+ "AutoPipelineForText2Image",
80
+ "AutoPipelineForImage2Image",
81
+ "AutoPipelineForInpainting",]
82
+
83
+ flax_list = ["FlaxDiffusionPipeline",]
84
+
85
+ if isinstance(Target_class,str):
86
+ Target_class = getattr(diffusers, Target_class)
87
+
88
+ cls_method= self.get_inherited_class(Target_class)
89
+
90
+ if any(method in torch_list for method in cls_method):
91
+ class_type= "torch"
92
+ elif any(method in flax_list for method in cls_method):
93
+ class_type= "flax"
94
+ else:
95
+ class_type= "onnx"
96
+ return class_type
97
+
98
+
99
+ def key_check(self,keyword) -> bool:
100
+ global key_dict
101
+ if "key_dict" not in globals():
102
+ key_dict = {}
103
+ key = str(keyword)
104
+ key_in = False
105
+ if key in key_dict:
106
+ if keyword == key_dict[key]:
107
+ key_in = True
108
+ key_dict[key] = keyword
109
+ return key_in
110
+
111
+
112
+ def get_call_method(
113
+ self,
114
+ class_name,
115
+ method_name : str = '__call__'
116
+ ) ->list:
117
+ """
118
+ Acquire the arguments of the function specified by 'method_name'
119
+ for the class specified by 'class_name'
120
+ """
121
+ if isinstance(class_name,str):
122
+ class_name = getattr(getattr(diffusers, class_name),method_name)
123
+ parameters = inspect.signature(class_name).parameters
124
+ arg_names = []
125
+ for param in parameters.values():
126
+ arg_names.append(param.name)
127
+ return arg_names
128
+
129
+
130
+ def get_class_elements(
131
+ self,
132
+ search
133
+ ):
134
+ return list(search.__class__.__annotations__.keys())
135
+
136
+
137
+ def check_for_safetensors(
138
+ self,
139
+ path
140
+ ):
141
+ _ext = os.path.basename(path).split(".")[-1]
142
+ if _ext == "safetensors":
143
+ return True
144
+ else:
145
+ return False
146
+
147
+
148
+ def pipe_class_type(
149
+ self,
150
+ class_name
151
+ ):
152
+ """
153
+ Args:
154
+ class_name : class
155
+
156
+ Returns:
157
+ Literal['txt2img','img2img','txt2video']
158
+ """
159
+ _txt2img_method_list = [] #else
160
+ _img2img_method_list = ["image"]
161
+ _img2video_method_list = ["video_length","fps"]
162
+
163
+ call_method = self.get_call_method(class_name,method_name = '__call__')
164
+
165
+ if any(method in call_method for method in _img2video_method_list):
166
+ pipeline_type = "txt2video"
167
+ elif any(method in call_method for method in _img2img_method_list):
168
+ pipeline_type = "img2img"
169
+ else:
170
+ pipeline_type = "txt2img"
171
+ return pipeline_type
172
+
173
+
174
+ def import_on_str(
175
+ self,
176
+ desired_function_or_class,
177
+ module_name = ""
178
+ ):
179
+ if not module_name:
180
+ import_object = __import__(desired_function_or_class)
181
+ else:
182
+ import_object = getattr(__import__(module_name), desired_function_or_class)
183
+ return import_object
184
+
185
+
186
+ def max_temper(
187
+ self,
188
+ search_word,
189
+ search_list
190
+ ):
191
+ return difflib.get_close_matches(search_word, search_list,cutoff=0, n=1)
192
+
193
+
194
+ def sort_list_obj(
195
+ self,
196
+ list_obj,
197
+ need_txt
198
+ ):
199
+ sorted_list=[]
200
+ for module_obj in list_obj:
201
+ if need_txt.lower() in module_obj.lower():
202
+ sorted_list.append(module_obj)
203
+ return sorted_list
204
+
205
+
206
+ def sort_by_version(self,sorted_list) -> list:
207
+ """
208
+ Returns:
209
+ Sorted by version in order of newest to oldest
210
+ """
211
+ return natsorted(sorted_list,reverse = True)
212
+
213
+
@@ -0,0 +1 @@
1
+ from .Base_config import Basic_config