auto-diffusers 1.3.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,385 @@
1
+ import os
2
+ import requests
3
+ from requests import HTTPError
4
+ from tqdm.auto import tqdm
5
+
6
+ from ..setup.Base_config import Basic_config
7
+
8
+
9
+ class Civitai(Basic_config):
10
+ '''
11
+ Example:
12
+ item = requests.get("http://civitai.example").json
13
+ state_list = [{
14
+ "repo_name": item["name"],
15
+ "repo_id": item["id"],
16
+ "favoriteCount": item["stats"]["favoriteCount"],
17
+ "downloadCount": item["stats"]["downloadCount"],
18
+ "CreatorName": item["creator"]["username"],
19
+ "version_list": [{
20
+ "id": item["modelVersions"]["id"],
21
+ "name": item["modelVersions"]["name"],
22
+ "downloadCount": item["modelVersions"]["downloadCount"],
23
+ "files": [{
24
+ "filename": item["modelVersions"]["files"]["name"],
25
+ "file_id": item["modelVersions"]["files"]["id"],
26
+ "download_url": item["modelVersions"]["files"]["downloadUrl"],
27
+ }]
28
+ }]
29
+ }]
30
+ return:
31
+ state_list = {
32
+ "repo_name": item["name"],
33
+ "repo_id": item["id"],
34
+ "favoriteCount": item["stats"]["favoriteCount"],
35
+ "downloadCount": item["stats"]["downloadCount"],
36
+ "CreatorName": item["creator"]["username"],
37
+ "version_list": <file_list>
38
+ }
39
+ '''
40
+
41
+ base_civitai_dir = "/root/.cache/Civitai"
42
+ max_number_of_choices:int = 15
43
+ chunk_size:int = 1024
44
+
45
+ def __init__(self):
46
+ super().__init__()
47
+ self.path_dict = {}
48
+ self.save_file_name = ""
49
+
50
+
51
+ def civitai_download(
52
+ self,
53
+ seach_word,
54
+ auto,
55
+ model_type,
56
+ download=True):
57
+ """
58
+ Function to download models from civitai.
59
+
60
+ Parameters:
61
+ - seach_word(str): Search query string.
62
+ - auto(bool): Flag for automatic selection.
63
+ - model_type(str): Type of model to search for.
64
+ arg:[Checkpoint,
65
+ TextualInversion,
66
+ Hypernetwork,
67
+ AestheticGradient,
68
+ LORA,
69
+ Controlnet,
70
+ Poses
71
+ ]
72
+ - download(bool): Whether to download the model
73
+
74
+ Returns:
75
+
76
+ Local storage path if download is true,
77
+ model download URL if false
78
+ ---
79
+ (model_url:str, save_path:str)
80
+ ---
81
+ """
82
+
83
+ model_url, save_path = self.requests_civitai(
84
+ query=seach_word,
85
+ auto=auto,
86
+ model_type=model_type)
87
+ if download:
88
+ self.download_model(
89
+ url=model_url,
90
+ save_path=save_path)
91
+ return (model_url, self.civitai_save_path())
92
+ else:
93
+ return (model_url, model_url)
94
+
95
+
96
+ def download_model(self, url, save_path):
97
+ if not self.is_url_valid(url):
98
+ raise requests.HTTPError("URL is invalid.")
99
+
100
+ response = requests.get(url, stream=True)
101
+
102
+ try:
103
+ response.raise_for_status()
104
+ except requests.HTTPError:
105
+ raise requests.HTTPError(f"Invalid URL: {response.status_code}")
106
+
107
+ os.makedirs(os.path.dirname(save_path),exist_ok=True)
108
+
109
+ with tqdm.wrapattr(open(save_path, "wb"), "write",
110
+ miniters=1, desc="Downloading model",
111
+ total=int(response.headers.get('content-length', 0))) as fout:
112
+ for chunk in response.iter_content(chunk_size=4096):
113
+ fout.write(chunk)
114
+ self.logger.info(f"Downloaded file saved to {save_path}")
115
+
116
+
117
+ def repo_select_civitai(self, state: list, auto: bool, recursive: bool = True):
118
+ """
119
+ Set repository requests for Civitai.
120
+
121
+ Parameters:
122
+ - state (list): List of repository information.
123
+ - auto (bool): Flag for automatic selection.
124
+ - recursive (bool): Flag for recursion.
125
+
126
+ Returns:
127
+ - dict: Selected repository information.
128
+ """
129
+ if not state:
130
+ raise ValueError("state is empty")
131
+
132
+ if auto:
133
+ return max(state, key=lambda x: x['downloadCount'])
134
+ else:
135
+ sorted_list = sorted(state, key=lambda x: x['downloadCount'], reverse=True)
136
+ if recursive and self.max_number_of_choices < len(sorted_list):
137
+ Limit_choice = True
138
+ else:
139
+ Limit_choice = False
140
+
141
+ if recursive:
142
+ print("\n\n\033[34mThe following repo paths were found\033[0m")
143
+ else:
144
+ print("\n\n\n")
145
+
146
+ max_number = min(self.max_number_of_choices, len(sorted_list)) if recursive else len(sorted_list)
147
+ for number, states_dict in enumerate(sorted_list[:max_number]):
148
+ print(f"\033[34m{number + 1}. Repo_id: {states_dict['CreatorName']} / {states_dict['repo_name']}, download: {states_dict['downloadCount']}")
149
+
150
+ if Limit_choice:
151
+ max_number += 1
152
+ print(f"\033[34m{max_number}. Other than above")
153
+
154
+ while True:
155
+ try:
156
+ choice = int(input(f"choice repo [1~{max_number}]: "))
157
+ except ValueError:
158
+ print("\033[33mOnly natural numbers are valid.\033[34m")
159
+ continue
160
+
161
+ if Limit_choice and choice == max_number:
162
+ return self.repo_select_civitai(state=state, auto=auto, recursive=False)
163
+ elif 1 <= choice <= max_number:
164
+ self.path_dict["repo_id"] = sorted_list[choice - 1]["repo_id"]
165
+ return sorted_list[choice - 1]
166
+ else:
167
+ print(f"\033[33mPlease enter the numbers 1~{max_number}\033[34m")
168
+
169
+
170
+ def version_select_civitai(self, state, auto, recursive: bool = True):
171
+ """
172
+ Set model requests for Civitai.
173
+
174
+ Parameters:
175
+ - state: Model information state.
176
+ - auto: Flag for automatic selection.
177
+ - recursive (bool): Flag for recursion.
178
+
179
+ Returns:
180
+ - dict: Selected model information.
181
+ """
182
+ if not state:
183
+ raise ValueError("state is empty")
184
+
185
+ ver_list = sorted(state["version_list"], key=lambda x: x['downloadCount'], reverse=True)
186
+
187
+ if recursive and self.max_number_of_choices < len(ver_list):
188
+ Limit_choice = True
189
+ else:
190
+ Limit_choice = False
191
+
192
+ if auto:
193
+ result_dict = max(ver_list, key=lambda x: x['downloadCount'])
194
+ ver_files_list = self.sort_by_version(result_dict["files"])
195
+ return_dict = ver_files_list[0]
196
+ self.path_dict["version_id"] = return_dict["id"]
197
+ return return_dict
198
+ else:
199
+ if recursive:
200
+ print("\n\n\033[34mThe following model paths were found\033[0m")
201
+ else:
202
+ print("\n\n\n")
203
+
204
+ if len(ver_list) == 1:
205
+ return ver_list
206
+
207
+ max_number = min(self.max_number_of_choices, len(ver_list)) if recursive else len(ver_list)
208
+
209
+ for number_, state_dict_ in enumerate(ver_list[:max_number]):
210
+ print(f"\033[34m{number_ + 1}. model_version: {state_dict_['name']}, download: {state_dict_['downloadCount']}")
211
+
212
+ if Limit_choice:
213
+ max_number += 1
214
+ print(f"{max_number}. Other than above")
215
+
216
+
217
+ while True:
218
+ try:
219
+ choice = int(input("Select the model path to use: "))
220
+ except ValueError:
221
+ print("\033[33mOnly natural numbers are valid.\033[34m")
222
+ continue
223
+ if Limit_choice and choice == max_number:
224
+ return self.version_select_civitai(state=state, auto=auto, recursive=False)
225
+ elif 1 <= choice <= max_number:
226
+ return_dict = ver_list[choice - 1]
227
+ self.path_dict["version_id"] = return_dict["id"]
228
+ return return_dict["files"]
229
+ else:
230
+ print(f"\033[33mPlease enter the numbers 1~{max_number}\033[34m")
231
+
232
+
233
+ def file_select_civitai(self, state_list, auto,recursive:bool=True):
234
+ """
235
+ Return the download URL for the selected file.
236
+
237
+ Parameters:
238
+ - state_list: List of file information.
239
+ - auto: Flag for automatic selection.
240
+
241
+ Returns:
242
+ - str: Download URL of the selected file.
243
+ """
244
+ if recursive and self.max_number_of_choices < len(state_list):
245
+ Limit_choice = True
246
+ else:
247
+ Limit_choice = False
248
+
249
+ if len(state_list) > 1 and (not auto):
250
+ max_number = min(self.max_number_of_choices, len(state_list)) if recursive else len(state_list)
251
+ for number, states_dict in enumerate(state_list[:max_number]):
252
+ print(f"\033[34m{number + 1}. File_name: {states_dict['filename']}")
253
+
254
+ if Limit_choice:
255
+ max_number += 1
256
+ print(f"{max_number}. Other than above")
257
+
258
+ while True:
259
+ try:
260
+ choice = int(input(f"Select the file to download[1~{max_number}]: "))
261
+ except ValueError:
262
+ print("\033[33mOnly natural numbers are valid.\033[34m")
263
+ continue
264
+ if Limit_choice and choice == max_number:
265
+ return self.file_select_civitai(state_list=state_list, auto=auto, recursive=False)
266
+ elif 1 <= choice <= len(state_list):
267
+ self.path_dict["filename"] = state_list[choice - 1]["filename"]
268
+ return state_list[choice - 1]
269
+ else:
270
+ print(f"\033[33mPlease enter the numbers 1~{len(state_list)}\033[34m")
271
+ else:
272
+ self.path_dict["filename"] = state_list[0]["filename"]
273
+ return state_list[0]
274
+
275
+
276
+ def civitai_save_path(self):
277
+ """
278
+ Set the save path using the information in path_dict.
279
+
280
+ Returns:
281
+ - str: Save path.
282
+ """
283
+ repo_level_dir = str(self.path_dict['repo_id'])
284
+ file_version_dir = str(self.path_dict['version_id'])
285
+ save_file_name = str(self.path_dict['filename'])
286
+ save_path = os.path.join(self.base_civitai_dir, repo_level_dir, file_version_dir, save_file_name)
287
+ return save_path
288
+
289
+
290
+ def requests_civitai(self, query, auto, model_type):
291
+ """
292
+ Fetch models from Civitai based on a query and model type.
293
+
294
+ Parameters:
295
+ - query: Search query string.
296
+ - auto: Flag for automatic selection.
297
+ - model_type: Type of model to search for.
298
+ arg:[Checkpoint,
299
+ TextualInversion,
300
+ Hypernetwork,
301
+ AestheticGradient,
302
+ LORA,
303
+ Controlnet,
304
+ Poses
305
+ ]
306
+
307
+ Returns:
308
+ - str: Download URL of the selected file.
309
+ (url, save_path)
310
+ """
311
+ state = []
312
+ repo_list = []
313
+ model_ver_list = []
314
+ version_dict = {}
315
+
316
+ params = {"query": query, "types": model_type, "sort": "Most Downloaded"}
317
+
318
+ try:
319
+ response = requests.get("https://civitai.com/api/v1/models", params=params)
320
+ response.raise_for_status()
321
+ except requests.exceptions.HTTPError as err:
322
+ raise HTTPError(f"Could not get elements from the URL. {err}")
323
+ else:
324
+ try:
325
+ data = response.json()
326
+ except AttributeError:
327
+ raise ValueError("Invalid JSON response")
328
+
329
+ items = data["items"]
330
+
331
+ for item in items:
332
+ for model_ver in item["modelVersions"]:
333
+ files_list = []
334
+
335
+ for model_value in model_ver["files"]:
336
+ if any(check_word in model_value for check_word in ["downloadUrl", "name"]):
337
+ file_status = {
338
+ "filename": model_value["name"],
339
+ "file_id": model_value["id"],
340
+ "download_url": model_value["downloadUrl"],
341
+ }
342
+ files_list.append(file_status)
343
+
344
+ version_dict = {
345
+ "id": model_ver["id"],
346
+ "name": model_ver["name"],
347
+ "downloadCount": model_ver["stats"]["downloadCount"],
348
+ "files": files_list,
349
+ }
350
+
351
+ if files_list:
352
+ model_ver_list.append(version_dict)
353
+
354
+ if all(check_txt in item.keys() for check_txt in ["name", "stats", "creator"]):
355
+ state_dict = {
356
+ "repo_name": item["name"],
357
+ "repo_id": item["id"],
358
+ "favoriteCount": item["stats"]["favoriteCount"],
359
+ "downloadCount": item["stats"]["downloadCount"],
360
+ "CreatorName": item["creator"]["username"],
361
+ "version_list": model_ver_list,
362
+ }
363
+
364
+ if model_ver_list:
365
+ state.append(state_dict)
366
+
367
+ if not state:
368
+ raise ValueError("No matches found for your criteria")
369
+
370
+ model_dict = self.repo_select_civitai(
371
+ state = state,
372
+ auto = auto
373
+ )
374
+ files_list = self.version_select_civitai(
375
+ state = model_dict,
376
+ auto = auto
377
+ )
378
+
379
+ file_status_dict = self.file_select_civitai(
380
+ state_list = files_list,
381
+ auto = auto)
382
+
383
+ save_path = self.civitai_save_path()
384
+
385
+ return (file_status_dict["download_url"], save_path)