pygpt-net 2.4.42__py3-none-any.whl → 2.4.44__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CHANGELOG.md +11 -0
- README.md +17 -2
- pygpt_net/CHANGELOG.txt +11 -0
- pygpt_net/__init__.py +3 -3
- pygpt_net/controller/attachment.py +31 -3
- pygpt_net/controller/chat/attachment.py +37 -36
- pygpt_net/controller/config/placeholder.py +6 -4
- pygpt_net/controller/idx/common.py +7 -3
- pygpt_net/core/attachments/__init__.py +7 -2
- pygpt_net/core/attachments/context.py +52 -34
- pygpt_net/core/db/__init__.py +2 -1
- pygpt_net/core/debug/attachments.py +1 -0
- pygpt_net/core/idx/__init__.py +8 -3
- pygpt_net/core/idx/indexing.py +24 -7
- pygpt_net/core/idx/ui/__init__.py +22 -0
- pygpt_net/core/idx/ui/loaders.py +217 -0
- pygpt_net/data/config/config.json +4 -4
- pygpt_net/data/config/models.json +3 -3
- pygpt_net/data/config/modes.json +3 -3
- pygpt_net/data/config/settings.json +5 -5
- pygpt_net/data/locale/locale.de.ini +3 -3
- pygpt_net/data/locale/locale.en.ini +11 -9
- pygpt_net/data/locale/locale.es.ini +3 -3
- pygpt_net/data/locale/locale.fr.ini +3 -3
- pygpt_net/data/locale/locale.it.ini +3 -3
- pygpt_net/data/locale/locale.pl.ini +3 -3
- pygpt_net/data/locale/locale.uk.ini +3 -3
- pygpt_net/data/locale/locale.zh.ini +3 -3
- pygpt_net/data/locale/plugin.mailer.en.ini +5 -5
- pygpt_net/item/attachment.py +5 -1
- pygpt_net/item/ctx.py +99 -2
- pygpt_net/migrations/Version20241215110000.py +25 -0
- pygpt_net/migrations/__init__.py +3 -1
- pygpt_net/plugin/cmd_files/__init__.py +3 -2
- pygpt_net/provider/core/attachment/json_file.py +4 -1
- pygpt_net/provider/core/config/patch.py +6 -0
- pygpt_net/provider/core/ctx/db_sqlite/storage.py +50 -7
- pygpt_net/provider/core/ctx/db_sqlite/utils.py +29 -5
- pygpt_net/provider/loaders/base.py +14 -0
- pygpt_net/provider/loaders/hub/yt/base.py +5 -0
- pygpt_net/provider/loaders/web_database.py +13 -5
- pygpt_net/provider/loaders/web_github_issues.py +5 -1
- pygpt_net/provider/loaders/web_google_calendar.py +9 -1
- pygpt_net/provider/loaders/web_google_docs.py +6 -1
- pygpt_net/provider/loaders/web_google_drive.py +10 -1
- pygpt_net/provider/loaders/web_google_gmail.py +2 -1
- pygpt_net/provider/loaders/web_google_keep.py +5 -1
- pygpt_net/provider/loaders/web_google_sheets.py +5 -1
- pygpt_net/provider/loaders/web_microsoft_onedrive.py +15 -1
- pygpt_net/provider/loaders/web_page.py +4 -2
- pygpt_net/provider/loaders/web_rss.py +2 -1
- pygpt_net/provider/loaders/web_sitemap.py +2 -1
- pygpt_net/provider/loaders/web_twitter.py +4 -2
- pygpt_net/provider/loaders/web_yt.py +17 -2
- pygpt_net/provider/vector_stores/ctx_attachment.py +1 -1
- pygpt_net/tools/indexer/__init__.py +8 -40
- pygpt_net/tools/indexer/ui/web.py +20 -78
- pygpt_net/ui/layout/ctx/ctx_list.py +86 -18
- pygpt_net/ui/widget/dialog/url.py +151 -14
- pygpt_net/ui/widget/element/group.py +15 -2
- pygpt_net/ui/widget/lists/context.py +23 -9
- pygpt_net/utils.py +1 -1
- {pygpt_net-2.4.42.dist-info → pygpt_net-2.4.44.dist-info}/METADATA +18 -3
- {pygpt_net-2.4.42.dist-info → pygpt_net-2.4.44.dist-info}/RECORD +67 -64
- {pygpt_net-2.4.42.dist-info → pygpt_net-2.4.44.dist-info}/LICENSE +0 -0
- {pygpt_net-2.4.42.dist-info → pygpt_net-2.4.44.dist-info}/WHEEL +0 -0
- {pygpt_net-2.4.42.dist-info → pygpt_net-2.4.44.dist-info}/entry_points.txt +0 -0
@@ -6,7 +6,7 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.12.
|
9
|
+
# Updated Date: 2024.12.16 01:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import copy
|
@@ -15,7 +15,9 @@ import shutil
|
|
15
15
|
import uuid
|
16
16
|
|
17
17
|
from shutil import copyfile
|
18
|
-
from typing import Optional, List, Dict, Any
|
18
|
+
from typing import Optional, List, Dict, Any, Tuple
|
19
|
+
|
20
|
+
from llama_index.core import Document
|
19
21
|
|
20
22
|
from pygpt_net.core.bridge import BridgeContext
|
21
23
|
from pygpt_net.core.events import KernelEvent
|
@@ -125,7 +127,7 @@ class Context:
|
|
125
127
|
meta_path = self.get_dir(meta)
|
126
128
|
context = ""
|
127
129
|
if os.path.exists(meta_path) and os.path.isdir(meta_path):
|
128
|
-
for file in meta.
|
130
|
+
for file in meta.get_additional_ctx():
|
129
131
|
if ("type" not in file
|
130
132
|
or file["type"] not in ["local_file", "url"]):
|
131
133
|
continue
|
@@ -183,7 +185,7 @@ class Context:
|
|
183
185
|
|
184
186
|
indexed = False
|
185
187
|
# index files if not indexed by auto_index
|
186
|
-
for i, file in enumerate(meta.
|
188
|
+
for i, file in enumerate(meta.get_additional_ctx()):
|
187
189
|
if "indexed" not in file or not file["indexed"]:
|
188
190
|
file_id = file["uuid"]
|
189
191
|
file_idx_path = os.path.join(meta_path, file_id)
|
@@ -313,7 +315,7 @@ class Context:
|
|
313
315
|
prompt: str,
|
314
316
|
auto_index: bool = False,
|
315
317
|
real_path: Optional[str] = None
|
316
|
-
) ->
|
318
|
+
) -> Dict[str, Any]:
|
317
319
|
"""
|
318
320
|
Upload attachment for context
|
319
321
|
|
@@ -325,7 +327,10 @@ class Context:
|
|
325
327
|
:return: Dict with attachment data
|
326
328
|
"""
|
327
329
|
if self.is_verbose():
|
328
|
-
|
330
|
+
if meta.group:
|
331
|
+
print("Uploading for meta group ID: {}".format(meta.group.id))
|
332
|
+
else:
|
333
|
+
print("Uploading for meta ID: {}".format(meta.id))
|
329
334
|
|
330
335
|
# prepare idx dir
|
331
336
|
name = os.path.basename(attachment.path)
|
@@ -342,11 +347,18 @@ class Context:
|
|
342
347
|
if auto_index:
|
343
348
|
print("Attachments: vector index path: {}".format(index_path))
|
344
349
|
|
345
|
-
|
346
|
-
|
350
|
+
documents = None
|
351
|
+
|
352
|
+
# store content to read, and get docs if type == web
|
353
|
+
src_file, docs = self.store_content(attachment, file_idx_path)
|
354
|
+
if attachment.type == AttachmentItem.TYPE_URL:
|
355
|
+
documents = docs
|
356
|
+
|
357
|
+
# extract text content using data loader, and get docs if type == file
|
358
|
+
content, docs = self.read_content(attachment, src_file, prompt)
|
359
|
+
if attachment.type == AttachmentItem.TYPE_FILE:
|
360
|
+
documents = docs
|
347
361
|
|
348
|
-
# extract text content using data loader
|
349
|
-
content = self.read_content(attachment, src_file, prompt)
|
350
362
|
if content:
|
351
363
|
text_path = os.path.join(file_idx_path, file_id + ".txt")
|
352
364
|
with open(text_path, "w", encoding="utf-8") as f:
|
@@ -372,7 +384,7 @@ class Context:
|
|
372
384
|
source = src_file
|
373
385
|
if attachment.type == AttachmentItem.TYPE_URL:
|
374
386
|
source = attachment.path # URL
|
375
|
-
doc_ids = self.index_attachment(attachment.type, source, index_path)
|
387
|
+
doc_ids = self.index_attachment(attachment.type, source, index_path, documents=documents)
|
376
388
|
|
377
389
|
result = {
|
378
390
|
"name": name,
|
@@ -402,57 +414,61 @@ class Context:
|
|
402
414
|
attachment: AttachmentItem,
|
403
415
|
path: str,
|
404
416
|
prompt: str
|
405
|
-
) -> str:
|
417
|
+
) -> Tuple[str, List[Document]]:
|
406
418
|
"""
|
407
419
|
Read content from attachment
|
408
420
|
|
409
421
|
:param attachment: AttachmentItem instance
|
410
422
|
:param path: source file path
|
411
423
|
:param prompt: user input prompt
|
412
|
-
:return: content
|
424
|
+
:return: text content, list of documents
|
413
425
|
"""
|
414
426
|
content = ""
|
427
|
+
docs = []
|
415
428
|
if attachment.type == AttachmentItem.TYPE_FILE:
|
416
429
|
loader_kwargs = {
|
417
430
|
"prompt": prompt,
|
418
431
|
} # extra loader kwargs
|
419
|
-
content = self.window.core.idx.indexing.read_text_content(
|
432
|
+
content, docs = self.window.core.idx.indexing.read_text_content(
|
420
433
|
path=path,
|
421
434
|
loader_kwargs=loader_kwargs,
|
422
435
|
)
|
423
436
|
elif attachment.type == AttachmentItem.TYPE_URL:
|
424
|
-
# directly from path
|
437
|
+
# directly from before stored path
|
425
438
|
with open(path, "r", encoding="utf-8") as f:
|
426
|
-
content = f.read() # already crawled
|
427
|
-
|
428
|
-
return content
|
439
|
+
content = f.read() # data is already crawled in `store_content`
|
440
|
+
return content, docs
|
429
441
|
|
430
442
|
def store_content(
|
431
443
|
self,
|
432
444
|
attachment: AttachmentItem,
|
433
445
|
dir: str
|
434
|
-
) -> str:
|
446
|
+
) -> Tuple[str, List[Document]]:
|
435
447
|
"""
|
436
448
|
Prepare content for attachment
|
437
449
|
|
438
450
|
:param attachment: AttachmentItem instance
|
439
451
|
:param dir: directory to save content
|
440
|
-
:return:
|
452
|
+
:return: path, list of documents
|
441
453
|
"""
|
442
454
|
path = None
|
455
|
+
docs = []
|
443
456
|
if attachment.type == AttachmentItem.TYPE_FILE:
|
444
|
-
# copy raw file
|
457
|
+
# copy raw file only
|
445
458
|
name = os.path.basename(attachment.path)
|
446
459
|
path = os.path.join(dir, name)
|
447
460
|
if os.path.exists(path):
|
448
461
|
os.remove(path)
|
449
462
|
copyfile(attachment.path, path)
|
450
463
|
elif attachment.type == AttachmentItem.TYPE_URL:
|
451
|
-
|
452
|
-
|
453
|
-
|
454
|
-
|
455
|
-
|
464
|
+
loader = attachment.extra.get("loader")
|
465
|
+
input_params = attachment.extra.get("input_params")
|
466
|
+
input_config = attachment.extra.get("input_config")
|
467
|
+
self.window.core.idx.indexing.update_loader_args(loader, input_config) # update config
|
468
|
+
content, docs = self.window.core.idx.indexing.read_web_content(
|
469
|
+
url="",
|
470
|
+
type=loader,
|
471
|
+
extra_args=input_params,
|
456
472
|
)
|
457
473
|
# src file save
|
458
474
|
name = "url.txt"
|
@@ -461,14 +477,14 @@ class Context:
|
|
461
477
|
os.remove(path)
|
462
478
|
with open(path, "w", encoding="utf-8") as f:
|
463
479
|
f.write(content)
|
464
|
-
return path
|
480
|
+
return path, docs
|
465
481
|
|
466
482
|
def index_attachment(
|
467
483
|
self,
|
468
484
|
type: str,
|
469
485
|
source: str,
|
470
486
|
idx_path: str,
|
471
|
-
documents: Optional[
|
487
|
+
documents: Optional[List[Document]] = None
|
472
488
|
) -> list:
|
473
489
|
"""
|
474
490
|
Index attachment
|
@@ -496,7 +512,7 @@ class Context:
|
|
496
512
|
:param meta: CtxMeta instance
|
497
513
|
:return: list of attachments
|
498
514
|
"""
|
499
|
-
return meta.
|
515
|
+
return meta.get_additional_ctx()
|
500
516
|
|
501
517
|
def get_dir(self, meta: CtxMeta) -> str:
|
502
518
|
"""
|
@@ -506,6 +522,8 @@ class Context:
|
|
506
522
|
:return: directory path
|
507
523
|
"""
|
508
524
|
meta_uuid = str(meta.uuid)
|
525
|
+
if meta.group:
|
526
|
+
meta_uuid = str(meta.group.uuid)
|
509
527
|
return os.path.join(self.window.core.config.get_user_dir("ctx_idx"), meta_uuid)
|
510
528
|
|
511
529
|
def get_selected_model(self, mode: str = "summary"):
|
@@ -562,7 +580,7 @@ class Context:
|
|
562
580
|
:param meta: CtxMeta instance
|
563
581
|
:return: number of attachments
|
564
582
|
"""
|
565
|
-
return len(meta.
|
583
|
+
return len(meta.get_additional_ctx())
|
566
584
|
|
567
585
|
def delete(
|
568
586
|
self,
|
@@ -577,11 +595,11 @@ class Context:
|
|
577
595
|
:param item: Attachment item dict
|
578
596
|
:param delete_files: delete files
|
579
597
|
"""
|
580
|
-
meta.
|
598
|
+
meta.remove_additional_ctx(item)
|
581
599
|
self.window.core.ctx.save(meta.id)
|
582
600
|
if delete_files:
|
583
601
|
self.delete_local(meta, item)
|
584
|
-
if len(meta.
|
602
|
+
if len(meta.get_additional_ctx()) == 0:
|
585
603
|
self.delete_index(meta)
|
586
604
|
|
587
605
|
def delete_by_meta(self, meta: CtxMeta):
|
@@ -613,7 +631,7 @@ class Context:
|
|
613
631
|
:param meta: CtxMeta instance
|
614
632
|
:param delete_files: delete files
|
615
633
|
"""
|
616
|
-
meta.
|
634
|
+
meta.reset_additional_ctx()
|
617
635
|
self.window.core.ctx.save(meta.id)
|
618
636
|
if delete_files:
|
619
637
|
self.delete_index(meta)
|
@@ -644,7 +662,7 @@ class Context:
|
|
644
662
|
:param meta: CtxMeta instance
|
645
663
|
:param delete_files: delete files
|
646
664
|
"""
|
647
|
-
meta.
|
665
|
+
meta.reset_additional_ctx()
|
648
666
|
self.window.core.ctx.save(meta.id)
|
649
667
|
if delete_files:
|
650
668
|
self.delete_index(meta)
|
pygpt_net/core/db/__init__.py
CHANGED
@@ -122,6 +122,7 @@ class Database:
|
|
122
122
|
'updated_ts',
|
123
123
|
'created_ts',
|
124
124
|
'uuid',
|
125
|
+
'additional_ctx_json',
|
125
126
|
]
|
126
127
|
columns["idx_ctx"] = [
|
127
128
|
'id',
|
@@ -242,7 +243,7 @@ class Database:
|
|
242
243
|
'sort_by': columns["ctx_group"],
|
243
244
|
'search_fields': ['id', 'name'],
|
244
245
|
'timestamp_columns': ['created_ts', 'updated_ts'],
|
245
|
-
'json_columns': [],
|
246
|
+
'json_columns': ['additional_ctx_json'],
|
246
247
|
'default_sort': 'id',
|
247
248
|
'default_order': 'DESC',
|
248
249
|
'primary_key': 'id',
|
pygpt_net/core/idx/__init__.py
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.12.
|
9
|
+
# Updated Date: 2024.12.16 01:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
@@ -24,6 +24,7 @@ from .indexing import Indexing
|
|
24
24
|
from .llm import Llm
|
25
25
|
from .chat import Chat
|
26
26
|
from .metadata import Metadata
|
27
|
+
from .ui import UI
|
27
28
|
|
28
29
|
from .types.ctx import Ctx
|
29
30
|
from .types.external import External
|
@@ -43,6 +44,7 @@ class Idx:
|
|
43
44
|
self.storage = Storage(window)
|
44
45
|
self.chat = Chat(window, self.storage)
|
45
46
|
self.metadata = Metadata(window)
|
47
|
+
self.ui = UI(window)
|
46
48
|
|
47
49
|
self.providers = {
|
48
50
|
"json_file": JsonFileProvider(window), # only for patching
|
@@ -529,7 +531,7 @@ class Idx:
|
|
529
531
|
self.items[store_id][idx].id = idx
|
530
532
|
self.items[store_id][idx].name = idx
|
531
533
|
|
532
|
-
def get_idx_ids(self) -> List[str]:
|
534
|
+
def get_idx_ids(self) -> List[Dict[str, str]]:
|
533
535
|
"""
|
534
536
|
Get list of indexes
|
535
537
|
|
@@ -539,7 +541,10 @@ class Idx:
|
|
539
541
|
data = self.window.core.config.get('llama.idx.list')
|
540
542
|
if data is not None:
|
541
543
|
for item in data:
|
542
|
-
|
544
|
+
name = item['name']
|
545
|
+
if name is None or name == "":
|
546
|
+
name = item['id']
|
547
|
+
ids.append({item['id']: name})
|
543
548
|
return ids
|
544
549
|
|
545
550
|
def clear(self, idx: str):
|
pygpt_net/core/idx/indexing.py
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.12.
|
9
|
+
# Updated Date: 2024.12.16 01:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
@@ -89,18 +89,35 @@ class Indexing:
|
|
89
89
|
"key": key,
|
90
90
|
"value": loader.init_args[key],
|
91
91
|
"type": "str", # default = str
|
92
|
+
"label": key,
|
93
|
+
"description": None,
|
92
94
|
}
|
93
95
|
# from config
|
94
96
|
if key in loader.args:
|
95
97
|
self.external_config[loader.id][key]["value"] = loader.args[key]
|
96
98
|
if key in loader.init_args_types:
|
97
99
|
self.external_config[loader.id][key]["type"] = loader.init_args_types[key]
|
100
|
+
if key in loader.init_args_labels:
|
101
|
+
self.external_config[loader.id][key]["label"] = loader.init_args_labels[key]
|
102
|
+
if key in loader.init_args_desc:
|
103
|
+
self.external_config[loader.id][key]["description"] = loader.init_args_desc[key]
|
98
104
|
|
99
105
|
except ImportError as e:
|
100
106
|
msg = "Error while registering data loader: " + loader.id + " - " + str(e)
|
101
107
|
self.window.core.debug.log(msg)
|
102
108
|
self.window.core.debug.log(e)
|
103
109
|
|
110
|
+
def get_loader(self, loader: str) -> Optional[BaseLoader]:
|
111
|
+
"""
|
112
|
+
Get data loader by id
|
113
|
+
|
114
|
+
:param loader: loader id
|
115
|
+
:return: data loader instance
|
116
|
+
"""
|
117
|
+
if loader in self.data_providers:
|
118
|
+
return self.data_providers[loader]
|
119
|
+
return None
|
120
|
+
|
104
121
|
def update_loader_args(
|
105
122
|
self,
|
106
123
|
loader: str,
|
@@ -344,13 +361,13 @@ class Indexing:
|
|
344
361
|
self,
|
345
362
|
path: str,
|
346
363
|
loader_kwargs: Optional[Dict[str, Any]] = None
|
347
|
-
) -> str:
|
364
|
+
) -> Tuple[str, List[Document]]:
|
348
365
|
"""
|
349
366
|
Get content from file using loaders
|
350
367
|
|
351
368
|
:param path: path to file
|
352
369
|
:param loader_kwargs: additional keyword arguments for data loader
|
353
|
-
:return:
|
370
|
+
:return: text content, list of documents
|
354
371
|
"""
|
355
372
|
docs = self.get_documents(
|
356
373
|
path,
|
@@ -361,27 +378,27 @@ class Indexing:
|
|
361
378
|
data = []
|
362
379
|
for doc in docs:
|
363
380
|
data.append(doc.text)
|
364
|
-
return "\n".join(data)
|
381
|
+
return "\n".join(data), docs
|
365
382
|
|
366
383
|
def read_web_content(
|
367
384
|
self,
|
368
385
|
url: str,
|
369
386
|
type: str = "webpage",
|
370
387
|
extra_args: Optional[Dict[str, Any]] = None
|
371
|
-
) -> str:
|
388
|
+
) -> Tuple[str, List[Document]]:
|
372
389
|
"""
|
373
390
|
Get content from external resource
|
374
391
|
|
375
392
|
:param url: external url to index
|
376
393
|
:param type: type of URL (webpage, feed, etc.)
|
377
394
|
:param extra_args: extra arguments for loader
|
378
|
-
:return:
|
395
|
+
:return: text content, list of documents
|
379
396
|
"""
|
380
397
|
docs = self.read_web(url, type, extra_args)
|
381
398
|
data = []
|
382
399
|
for doc in docs:
|
383
400
|
data.append(doc.text)
|
384
|
-
return "\n".join(data)
|
401
|
+
return "\n".join(data), docs
|
385
402
|
|
386
403
|
def read_web(
|
387
404
|
self,
|
@@ -0,0 +1,22 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# ================================================== #
|
4
|
+
# This file is a part of PYGPT package #
|
5
|
+
# Website: https://pygpt.net #
|
6
|
+
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
|
+
# MIT License #
|
8
|
+
# Created By : Marcin Szczygliński #
|
9
|
+
# Updated Date: 2024.12.16 01:00:00 #
|
10
|
+
# ================================================== #
|
11
|
+
|
12
|
+
from .loaders import Loaders
|
13
|
+
|
14
|
+
class UI:
|
15
|
+
def __init__(self, window=None):
|
16
|
+
"""
|
17
|
+
UI components
|
18
|
+
|
19
|
+
:param window: Window instance
|
20
|
+
"""
|
21
|
+
self.window = window
|
22
|
+
self.loaders = Loaders(window)
|
@@ -0,0 +1,217 @@
|
|
1
|
+
#!/usr/bin/env python3
|
2
|
+
# -*- coding: utf-8 -*-
|
3
|
+
# ================================================== #
|
4
|
+
# This file is a part of PYGPT package #
|
5
|
+
# Website: https://pygpt.net #
|
6
|
+
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
|
+
# MIT License #
|
8
|
+
# Created By : Marcin Szczygliński #
|
9
|
+
# Updated Date: 2024.12.16 01:00:00 #
|
10
|
+
# ================================================== #
|
11
|
+
|
12
|
+
import json
|
13
|
+
from typing import Dict, Tuple, Any, Optional
|
14
|
+
|
15
|
+
from PySide6.QtCore import Qt
|
16
|
+
from PySide6.QtWidgets import QVBoxLayout, QLabel, QHBoxLayout, QWidget
|
17
|
+
|
18
|
+
from pygpt_net.ui.widget.element.labels import HelpLabel
|
19
|
+
from pygpt_net.ui.widget.option.input import OptionInput
|
20
|
+
|
21
|
+
|
22
|
+
class Loaders:
|
23
|
+
def __init__(self, window=None):
|
24
|
+
"""
|
25
|
+
UI - loaders components
|
26
|
+
|
27
|
+
:param window: Window instance
|
28
|
+
"""
|
29
|
+
self.window = window
|
30
|
+
|
31
|
+
def handle_options(
|
32
|
+
self,
|
33
|
+
select_loader,
|
34
|
+
prefix_options,
|
35
|
+
prefix_config
|
36
|
+
) -> Tuple[bool, Optional[str], Dict[str, Any], Dict[str, Any]]:
|
37
|
+
"""
|
38
|
+
Handle options
|
39
|
+
|
40
|
+
:param select_loader: loader selection
|
41
|
+
:param prefix_options: prefix for options
|
42
|
+
:param prefix_config: prefix for config
|
43
|
+
:return: bool, loader name, input_params, input_config
|
44
|
+
"""
|
45
|
+
input_params = {}
|
46
|
+
input_config = {}
|
47
|
+
loader = select_loader.get_value()
|
48
|
+
if not loader:
|
49
|
+
return False, loader, input_params, input_config
|
50
|
+
loaders = self.window.core.idx.indexing.get_external_instructions()
|
51
|
+
if loader in loaders:
|
52
|
+
params = loaders[loader]
|
53
|
+
for k in params["args"]:
|
54
|
+
key_path = prefix_options + "." + loader + "." + k
|
55
|
+
if key_path in self.window.ui.nodes:
|
56
|
+
tmp_value = self.window.ui.nodes[key_path].text()
|
57
|
+
type = params["args"][k]["type"]
|
58
|
+
try:
|
59
|
+
if tmp_value:
|
60
|
+
if type == "int":
|
61
|
+
tmp_value = int(tmp_value)
|
62
|
+
elif type == "float":
|
63
|
+
tmp_value = float(tmp_value)
|
64
|
+
elif type == "bool":
|
65
|
+
if tmp_value.lower() in ["true", "1"]:
|
66
|
+
tmp_value = True
|
67
|
+
else:
|
68
|
+
tmp_value = False
|
69
|
+
elif type == "list":
|
70
|
+
tmp_value = tmp_value.split(",")
|
71
|
+
elif type == "dict":
|
72
|
+
tmp_value = json.loads(tmp_value)
|
73
|
+
input_params[k] = tmp_value
|
74
|
+
except Exception as e:
|
75
|
+
self.window.core.debug.log(e)
|
76
|
+
self.window.ui.dialogs.alert(e)
|
77
|
+
|
78
|
+
loaders = self.window.core.idx.indexing.get_external_config()
|
79
|
+
if loader in loaders:
|
80
|
+
params = loaders[loader]
|
81
|
+
for k in params:
|
82
|
+
key_path = prefix_config + "." + loader + "." + k
|
83
|
+
type = params[k]["type"]
|
84
|
+
if key_path in self.window.ui.nodes:
|
85
|
+
tmp_value = self.window.ui.nodes[key_path].text()
|
86
|
+
try:
|
87
|
+
if tmp_value:
|
88
|
+
if type == "int":
|
89
|
+
tmp_value = int(tmp_value)
|
90
|
+
elif type == "float":
|
91
|
+
tmp_value = float(tmp_value)
|
92
|
+
elif type == "bool":
|
93
|
+
if tmp_value.lower() in ["true", "1"]:
|
94
|
+
tmp_value = True
|
95
|
+
else:
|
96
|
+
tmp_value = False
|
97
|
+
elif type == "list":
|
98
|
+
tmp_value = tmp_value.split(",")
|
99
|
+
elif type == "dict":
|
100
|
+
tmp_value = json.loads(tmp_value)
|
101
|
+
input_config[k] = tmp_value
|
102
|
+
except Exception as e:
|
103
|
+
self.window.core.debug.log(e)
|
104
|
+
self.window.ui.dialogs.alert(e)
|
105
|
+
|
106
|
+
return True, loader, input_params, input_config
|
107
|
+
|
108
|
+
def setup_loader_options(self):
|
109
|
+
"""Setup loader options"""
|
110
|
+
inputs = {}
|
111
|
+
groups = {}
|
112
|
+
loaders = self.window.core.idx.indexing.get_external_instructions()
|
113
|
+
for loader in loaders:
|
114
|
+
params = loaders[loader]
|
115
|
+
inputs[loader] = {}
|
116
|
+
group = QVBoxLayout()
|
117
|
+
for k in params["args"]:
|
118
|
+
label = k
|
119
|
+
description = None
|
120
|
+
is_label = False
|
121
|
+
if "label" in params["args"][k]:
|
122
|
+
label = params["args"][k]["label"]
|
123
|
+
is_label = True
|
124
|
+
if "description" in params["args"][k]:
|
125
|
+
description = params["args"][k]["description"]
|
126
|
+
option_id = "web.loader." + loader + ".option." + k
|
127
|
+
option_widget = OptionInput(self.window, "tool.indexer", option_id, {
|
128
|
+
"label": label,
|
129
|
+
"value": "",
|
130
|
+
})
|
131
|
+
option_widget.setPlaceholderText(params["args"][k]["type"])
|
132
|
+
inputs[loader][k] = option_widget
|
133
|
+
|
134
|
+
option_label = QLabel(label)
|
135
|
+
option_label.setToolTip(k)
|
136
|
+
|
137
|
+
row = QHBoxLayout() # cols
|
138
|
+
row.addWidget(option_label)
|
139
|
+
row.addWidget(option_widget)
|
140
|
+
row.setContentsMargins(5, 0, 5, 0)
|
141
|
+
|
142
|
+
option_layout = QVBoxLayout()
|
143
|
+
option_layout.addLayout(row)
|
144
|
+
if description:
|
145
|
+
option_layout.addWidget(HelpLabel(description))
|
146
|
+
|
147
|
+
option_layout.setContentsMargins(5, 0, 0, 0)
|
148
|
+
|
149
|
+
group.addLayout(option_layout)
|
150
|
+
group.setContentsMargins(0, 0, 0, 0)
|
151
|
+
|
152
|
+
group_widget = QWidget()
|
153
|
+
group_widget.setLayout(group)
|
154
|
+
groups[loader] = group_widget
|
155
|
+
|
156
|
+
return inputs, groups
|
157
|
+
|
158
|
+
def setup_loader_config(self):
|
159
|
+
"""Setup loader config"""
|
160
|
+
inputs = {}
|
161
|
+
groups = {}
|
162
|
+
loaders = self.window.core.idx.indexing.get_external_config()
|
163
|
+
for loader in loaders:
|
164
|
+
params = loaders[loader]
|
165
|
+
inputs[loader] = {}
|
166
|
+
group = QVBoxLayout()
|
167
|
+
for k in params:
|
168
|
+
label = k
|
169
|
+
description = None
|
170
|
+
is_label = False
|
171
|
+
if "label" in params[k]:
|
172
|
+
label = params[k]["label"]
|
173
|
+
is_label = True
|
174
|
+
if "description" in params[k]:
|
175
|
+
description = params[k]["description"]
|
176
|
+
option_id = "web.loader." + loader + ".config." + k
|
177
|
+
option_widget = OptionInput(self.window, "tool.indexer", option_id, {
|
178
|
+
"label": label,
|
179
|
+
"value": params[k]["value"],
|
180
|
+
})
|
181
|
+
try:
|
182
|
+
if params[k]["value"] is not None:
|
183
|
+
if params[k]["type"] == "list" and isinstance(params[k]["value"], list):
|
184
|
+
option_widget.setText(", ".join(params[k]["value"]))
|
185
|
+
elif params[k]["type"] == "dict" and isinstance(params[k]["value"], dict):
|
186
|
+
option_widget.setText(json.dumps(params[k]["value"]))
|
187
|
+
else:
|
188
|
+
option_widget.setText(str(params[k]["value"]))
|
189
|
+
except Exception as e:
|
190
|
+
self.window.core.debug.log(e)
|
191
|
+
|
192
|
+
option_widget.setPlaceholderText(params[k]["type"])
|
193
|
+
inputs[loader][k] = option_widget
|
194
|
+
|
195
|
+
option_label = QLabel(label)
|
196
|
+
option_label.setToolTip(k)
|
197
|
+
|
198
|
+
row = QHBoxLayout() # cols
|
199
|
+
row.addWidget(option_label)
|
200
|
+
row.addWidget(option_widget)
|
201
|
+
row.setContentsMargins(5, 0, 5, 0)
|
202
|
+
|
203
|
+
option_layout = QVBoxLayout()
|
204
|
+
option_layout.addLayout(row)
|
205
|
+
if description:
|
206
|
+
option_layout.addWidget(HelpLabel(description))
|
207
|
+
|
208
|
+
option_layout.setContentsMargins(5, 0, 0, 0)
|
209
|
+
|
210
|
+
group.addLayout(option_layout)
|
211
|
+
group.setContentsMargins(0, 0, 0, 0)
|
212
|
+
|
213
|
+
group_widget = QWidget()
|
214
|
+
group_widget.setLayout(group)
|
215
|
+
groups[loader] = group_widget
|
216
|
+
|
217
|
+
return inputs, groups
|
@@ -1,8 +1,8 @@
|
|
1
1
|
{
|
2
2
|
"__meta__": {
|
3
|
-
"version": "2.4.
|
4
|
-
"app.version": "2.4.
|
5
|
-
"updated_at": "2024-12-
|
3
|
+
"version": "2.4.44",
|
4
|
+
"app.version": "2.4.44",
|
5
|
+
"updated_at": "2024-12-16T00:00:00"
|
6
6
|
},
|
7
7
|
"access.audio.event.speech": false,
|
8
8
|
"access.audio.event.speech.disabled": [],
|
@@ -100,7 +100,7 @@
|
|
100
100
|
6,
|
101
101
|
7
|
102
102
|
],
|
103
|
-
"ctx.records.folders.top":
|
103
|
+
"ctx.records.folders.top": true,
|
104
104
|
"ctx.records.limit": 0,
|
105
105
|
"ctx.records.separators": true,
|
106
106
|
"ctx.records.groups.separators": true,
|