pygpt-net 2.4.36.post1__py3-none-any.whl → 2.4.41__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- CHANGELOG.md +38 -1
- README.md +71 -114
- pygpt_net/CHANGELOG.txt +38 -1
- pygpt_net/__init__.py +3 -3
- pygpt_net/controller/__init__.py +7 -3
- pygpt_net/controller/access/control.py +1 -1
- pygpt_net/controller/access/voice.py +11 -5
- pygpt_net/controller/agent/experts.py +11 -6
- pygpt_net/controller/agent/legacy.py +8 -6
- pygpt_net/controller/agent/llama.py +4 -2
- pygpt_net/controller/assistant/__init__.py +9 -4
- pygpt_net/controller/assistant/batch.py +38 -21
- pygpt_net/controller/assistant/editor.py +7 -6
- pygpt_net/controller/assistant/files.py +23 -7
- pygpt_net/controller/assistant/store.py +20 -7
- pygpt_net/controller/assistant/threads.py +34 -8
- pygpt_net/controller/attachment.py +29 -10
- pygpt_net/controller/audio/__init__.py +25 -4
- pygpt_net/controller/calendar/__init__.py +23 -4
- pygpt_net/controller/calendar/note.py +57 -11
- pygpt_net/controller/camera.py +3 -2
- pygpt_net/controller/chat/__init__.py +5 -3
- pygpt_net/controller/chat/attachment.py +39 -44
- pygpt_net/controller/chat/command.py +4 -2
- pygpt_net/controller/chat/common.py +11 -4
- pygpt_net/controller/chat/files.py +10 -3
- pygpt_net/controller/chat/image.py +17 -5
- pygpt_net/controller/chat/input.py +10 -7
- pygpt_net/controller/chat/output.py +21 -6
- pygpt_net/controller/chat/render.py +100 -21
- pygpt_net/controller/chat/response.py +34 -7
- pygpt_net/controller/chat/stream.py +4 -2
- pygpt_net/controller/chat/text.py +6 -4
- pygpt_net/controller/command.py +11 -3
- pygpt_net/controller/config/__init__.py +34 -6
- pygpt_net/controller/config/field/checkbox.py +7 -4
- pygpt_net/controller/config/field/cmd.py +7 -5
- pygpt_net/controller/config/field/combo.py +14 -6
- pygpt_net/controller/config/field/dictionary.py +14 -11
- pygpt_net/controller/config/field/input.py +9 -6
- pygpt_net/controller/config/field/slider.py +11 -8
- pygpt_net/controller/config/field/textarea.py +8 -5
- pygpt_net/controller/config/placeholder.py +52 -21
- pygpt_net/controller/ctx/__init__.py +138 -49
- pygpt_net/controller/ctx/common.py +15 -4
- pygpt_net/controller/ctx/extra.py +11 -3
- pygpt_net/controller/ctx/summarizer.py +24 -5
- pygpt_net/controller/debug/__init__.py +27 -6
- pygpt_net/controller/dialogs/confirm.py +34 -7
- pygpt_net/controller/dialogs/debug.py +4 -2
- pygpt_net/controller/dialogs/info.py +7 -2
- pygpt_net/controller/files.py +48 -10
- pygpt_net/controller/finder.py +11 -5
- pygpt_net/controller/idx/__init__.py +10 -3
- pygpt_net/controller/idx/common.py +4 -2
- pygpt_net/controller/idx/indexer.py +25 -17
- pygpt_net/controller/idx/settings.py +9 -3
- pygpt_net/controller/kernel/__init__.py +34 -8
- pygpt_net/controller/kernel/reply.py +12 -3
- pygpt_net/controller/kernel/stack.py +5 -3
- pygpt_net/controller/lang/custom.py +2 -7
- pygpt_net/controller/lang/mapping.py +5 -3
- pygpt_net/controller/layout.py +2 -2
- pygpt_net/controller/mode.py +16 -4
- pygpt_net/controller/model/__init__.py +14 -3
- pygpt_net/controller/model/editor.py +8 -3
- pygpt_net/controller/notepad.py +26 -12
- pygpt_net/controller/painter/capture.py +23 -4
- pygpt_net/controller/painter/common.py +9 -7
- pygpt_net/controller/plugins/__init__.py +19 -5
- pygpt_net/controller/plugins/presets.py +15 -6
- pygpt_net/controller/plugins/settings.py +9 -3
- pygpt_net/controller/presets/__init__.py +55 -16
- pygpt_net/controller/presets/editor.py +26 -10
- pygpt_net/controller/settings/__init__.py +3 -2
- pygpt_net/controller/settings/editor.py +29 -7
- pygpt_net/controller/settings/profile.py +22 -5
- pygpt_net/controller/theme/__init__.py +54 -12
- pygpt_net/controller/theme/common.py +24 -2
- pygpt_net/controller/theme/markdown.py +32 -16
- pygpt_net/controller/theme/menu.py +26 -5
- pygpt_net/controller/theme/nodes.py +2 -5
- pygpt_net/controller/tools/__init__.py +40 -2
- pygpt_net/controller/ui/__init__.py +4 -6
- pygpt_net/controller/ui/tabs.py +363 -65
- pygpt_net/core/access/actions.py +6 -4
- pygpt_net/core/access/shortcuts.py +4 -3
- pygpt_net/core/access/voice.py +6 -5
- pygpt_net/core/agents/legacy.py +4 -2
- pygpt_net/core/agents/memory.py +7 -2
- pygpt_net/core/agents/observer/evaluation.py +15 -7
- pygpt_net/core/agents/provider.py +9 -4
- pygpt_net/core/agents/runner.py +61 -15
- pygpt_net/core/agents/tools.py +23 -5
- pygpt_net/core/assistants/__init__.py +6 -4
- pygpt_net/core/assistants/files.py +35 -12
- pygpt_net/core/assistants/store.py +20 -10
- pygpt_net/core/attachments/__init__.py +54 -15
- pygpt_net/core/attachments/context.py +217 -64
- pygpt_net/core/audio/__init__.py +71 -3
- pygpt_net/core/audio/context.py +7 -2
- pygpt_net/core/bridge/__init__.py +22 -6
- pygpt_net/core/bridge/context.py +5 -3
- pygpt_net/core/bridge/worker.py +16 -2
- pygpt_net/core/calendar/__init__.py +57 -11
- pygpt_net/core/chain/__init__.py +8 -2
- pygpt_net/core/chain/chat.py +10 -8
- pygpt_net/core/chain/completion.py +10 -7
- pygpt_net/core/command.py +62 -17
- pygpt_net/core/ctx/__init__.py +260 -58
- pygpt_net/core/ctx/bag.py +25 -4
- pygpt_net/core/ctx/container.py +28 -17
- pygpt_net/core/ctx/idx.py +45 -8
- pygpt_net/core/ctx/output.py +95 -74
- pygpt_net/core/ctx/reply.py +5 -2
- pygpt_net/core/db/__init__.py +8 -7
- pygpt_net/core/db/viewer.py +17 -11
- pygpt_net/core/debug/__init__.py +10 -9
- pygpt_net/core/debug/tabs.py +5 -2
- pygpt_net/core/docker/__init__.py +11 -5
- pygpt_net/core/docker/builder.py +11 -3
- pygpt_net/core/events/app.py +5 -3
- pygpt_net/core/events/base.py +11 -5
- pygpt_net/core/events/control.py +5 -3
- pygpt_net/core/events/event.py +18 -7
- pygpt_net/core/events/kernel.py +5 -3
- pygpt_net/core/events/render.py +5 -3
- pygpt_net/core/experts/__init__.py +5 -4
- pygpt_net/core/filesystem/__init__.py +52 -34
- pygpt_net/core/filesystem/actions.py +8 -5
- pygpt_net/core/filesystem/editor.py +13 -3
- pygpt_net/core/filesystem/types.py +12 -7
- pygpt_net/core/filesystem/url.py +7 -3
- pygpt_net/core/idx/__init__.py +34 -25
- pygpt_net/core/idx/chat.py +60 -38
- pygpt_net/core/idx/context.py +6 -2
- pygpt_net/core/idx/indexing.py +84 -35
- pygpt_net/core/idx/llm.py +11 -3
- pygpt_net/core/idx/metadata.py +13 -3
- pygpt_net/core/idx/types/ctx.py +32 -6
- pygpt_net/core/idx/types/external.py +41 -7
- pygpt_net/core/idx/types/files.py +31 -6
- pygpt_net/core/image.py +15 -4
- pygpt_net/core/llm/__init__.py +13 -3
- pygpt_net/core/locale.py +34 -8
- pygpt_net/core/models.py +4 -3
- pygpt_net/core/notepad.py +9 -4
- pygpt_net/core/plugins.py +7 -6
- pygpt_net/core/presets.py +19 -10
- pygpt_net/core/profile.py +12 -6
- pygpt_net/core/prompt/__init__.py +10 -3
- pygpt_net/core/prompt/custom.py +7 -6
- pygpt_net/core/prompt/template.py +9 -3
- pygpt_net/core/render/base.py +117 -22
- pygpt_net/core/render/markdown/body.py +27 -7
- pygpt_net/core/render/markdown/renderer.py +119 -22
- pygpt_net/core/render/plain/body.py +22 -5
- pygpt_net/core/render/plain/renderer.py +97 -21
- pygpt_net/core/render/web/body.py +75 -25
- pygpt_net/core/render/web/renderer.py +313 -63
- pygpt_net/core/settings.py +9 -4
- pygpt_net/core/tabs/__init__.py +290 -103
- pygpt_net/core/tabs/tab.py +17 -4
- pygpt_net/core/tokens.py +44 -11
- pygpt_net/core/updater/__init__.py +20 -7
- pygpt_net/core/vision/analyzer.py +29 -6
- pygpt_net/core/web.py +130 -2
- pygpt_net/data/config/config.json +19 -6
- pygpt_net/data/config/models.json +3 -3
- pygpt_net/data/config/modes.json +3 -3
- pygpt_net/data/config/settings.json +81 -10
- pygpt_net/data/config/settings_section.json +3 -0
- pygpt_net/data/css/style.light.css +1 -0
- pygpt_net/data/css/{web.css → web-blocks.css} +162 -133
- pygpt_net/data/css/{web.light.css → web-blocks.light.css} +7 -0
- pygpt_net/data/css/web-chatgpt.css +350 -0
- pygpt_net/data/css/web-chatgpt.dark.css +64 -0
- pygpt_net/data/css/web-chatgpt.light.css +75 -0
- pygpt_net/data/css/web-chatgpt_wide.css +350 -0
- pygpt_net/data/css/web-chatgpt_wide.dark.css +64 -0
- pygpt_net/data/css/web-chatgpt_wide.light.css +75 -0
- pygpt_net/data/icons/split_screen.svg +1 -0
- pygpt_net/data/locale/locale.de.ini +19 -3
- pygpt_net/data/locale/locale.en.ini +30 -11
- pygpt_net/data/locale/locale.es.ini +19 -3
- pygpt_net/data/locale/locale.fr.ini +19 -3
- pygpt_net/data/locale/locale.it.ini +19 -3
- pygpt_net/data/locale/locale.pl.ini +20 -4
- pygpt_net/data/locale/locale.uk.ini +19 -3
- pygpt_net/data/locale/locale.zh.ini +20 -4
- pygpt_net/data/locale/plugin.cmd_web.de.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.en.ini +20 -10
- pygpt_net/data/locale/plugin.cmd_web.es.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.fr.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.it.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.pl.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.uk.ini +2 -0
- pygpt_net/data/locale/plugin.cmd_web.zh.ini +2 -0
- pygpt_net/icons.qrc +1 -0
- pygpt_net/icons_rc.py +165 -136
- pygpt_net/item/ctx.py +46 -24
- pygpt_net/plugin/audio_input/simple.py +21 -5
- pygpt_net/plugin/audio_output/__init__.py +4 -1
- pygpt_net/plugin/base/config.py +4 -2
- pygpt_net/plugin/base/plugin.py +26 -6
- pygpt_net/plugin/base/worker.py +37 -9
- pygpt_net/plugin/cmd_code_interpreter/__init__.py +39 -37
- pygpt_net/plugin/cmd_code_interpreter/runner.py +25 -12
- pygpt_net/plugin/cmd_web/__init__.py +46 -6
- pygpt_net/plugin/cmd_web/config.py +74 -48
- pygpt_net/plugin/cmd_web/websearch.py +61 -28
- pygpt_net/plugin/cmd_web/worker.py +79 -13
- pygpt_net/plugin/idx_llama_index/__init__.py +2 -2
- pygpt_net/plugin/real_time/__init__.py +2 -2
- pygpt_net/provider/core/config/patch.py +54 -1
- pygpt_net/provider/core/ctx/base.py +4 -1
- pygpt_net/provider/core/ctx/db_sqlite/__init__.py +10 -1
- pygpt_net/provider/core/ctx/db_sqlite/storage.py +22 -1
- pygpt_net/provider/gpt/assistants.py +10 -9
- pygpt_net/provider/gpt/audio.py +3 -2
- pygpt_net/provider/gpt/chat.py +8 -7
- pygpt_net/provider/gpt/completion.py +6 -4
- pygpt_net/provider/gpt/image.py +9 -2
- pygpt_net/provider/gpt/store.py +14 -13
- pygpt_net/provider/gpt/vision.py +6 -5
- pygpt_net/tools/__init__.py +9 -1
- pygpt_net/tools/base.py +15 -1
- pygpt_net/tools/code_interpreter/__init__.py +174 -75
- pygpt_net/tools/code_interpreter/ui/dialogs.py +21 -103
- pygpt_net/tools/code_interpreter/ui/widgets.py +284 -9
- pygpt_net/tools/html_canvas/__init__.py +78 -23
- pygpt_net/tools/html_canvas/ui/dialogs.py +46 -62
- pygpt_net/tools/html_canvas/ui/widgets.py +96 -3
- pygpt_net/ui/base/context_menu.py +2 -2
- pygpt_net/ui/layout/chat/input.py +10 -18
- pygpt_net/ui/layout/chat/output.py +26 -44
- pygpt_net/ui/layout/ctx/ctx_list.py +13 -4
- pygpt_net/ui/layout/toolbox/footer.py +18 -2
- pygpt_net/ui/main.py +2 -2
- pygpt_net/ui/menu/config.py +7 -11
- pygpt_net/ui/menu/debug.py +11 -1
- pygpt_net/ui/menu/theme.py +9 -2
- pygpt_net/ui/widget/filesystem/explorer.py +2 -2
- pygpt_net/ui/widget/lists/context.py +27 -5
- pygpt_net/ui/widget/tabs/Input.py +2 -2
- pygpt_net/ui/widget/tabs/body.py +2 -1
- pygpt_net/ui/widget/tabs/layout.py +195 -0
- pygpt_net/ui/widget/tabs/output.py +218 -55
- pygpt_net/ui/widget/textarea/html.py +11 -1
- pygpt_net/ui/widget/textarea/output.py +10 -1
- pygpt_net/ui/widget/textarea/search_input.py +4 -1
- pygpt_net/ui/widget/textarea/web.py +49 -9
- {pygpt_net-2.4.36.post1.dist-info → pygpt_net-2.4.41.dist-info}/METADATA +72 -115
- {pygpt_net-2.4.36.post1.dist-info → pygpt_net-2.4.41.dist-info}/RECORD +258 -250
- /pygpt_net/data/css/{web.dark.css → web-blocks.dark.css} +0 -0
- {pygpt_net-2.4.36.post1.dist-info → pygpt_net-2.4.41.dist-info}/LICENSE +0 -0
- {pygpt_net-2.4.36.post1.dist-info → pygpt_net-2.4.41.dist-info}/WHEEL +0 -0
- {pygpt_net-2.4.36.post1.dist-info → pygpt_net-2.4.41.dist-info}/entry_points.txt +0 -0
pygpt_net/core/idx/indexing.py
CHANGED
@@ -6,7 +6,7 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 08:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
@@ -14,6 +14,8 @@ import os.path
|
|
14
14
|
import time
|
15
15
|
|
16
16
|
from pathlib import Path
|
17
|
+
from typing import Optional, Tuple, List, Dict, Any
|
18
|
+
|
17
19
|
from sqlalchemy import text
|
18
20
|
|
19
21
|
from llama_index.core.indices.base import BaseIndex
|
@@ -99,7 +101,11 @@ class Indexing:
|
|
99
101
|
self.window.core.debug.log(msg)
|
100
102
|
self.window.core.debug.log(e)
|
101
103
|
|
102
|
-
def update_loader_args(
|
104
|
+
def update_loader_args(
|
105
|
+
self,
|
106
|
+
loader: str,
|
107
|
+
args: Dict[str, Any]
|
108
|
+
):
|
103
109
|
"""
|
104
110
|
Update loader arguments
|
105
111
|
|
@@ -142,7 +148,7 @@ class Indexing:
|
|
142
148
|
self.register_loader(loader)
|
143
149
|
self.window.core.idx.log("Data loaders reloaded.")
|
144
150
|
|
145
|
-
def get_external_instructions(self) ->
|
151
|
+
def get_external_instructions(self) -> Dict[str, Any]:
|
146
152
|
"""
|
147
153
|
Get external instructions
|
148
154
|
|
@@ -150,7 +156,7 @@ class Indexing:
|
|
150
156
|
"""
|
151
157
|
return self.external_instructions
|
152
158
|
|
153
|
-
def get_external_config(self) ->
|
159
|
+
def get_external_config(self) -> Dict[str, Any]:
|
154
160
|
"""
|
155
161
|
Get external config
|
156
162
|
|
@@ -177,7 +183,7 @@ class Indexing:
|
|
177
183
|
if ext in extensions:
|
178
184
|
return loader["loader"]
|
179
185
|
|
180
|
-
def get_data_providers(self) ->
|
186
|
+
def get_data_providers(self) -> Dict[str, BaseLoader]:
|
181
187
|
"""
|
182
188
|
Get data providers
|
183
189
|
|
@@ -185,7 +191,11 @@ class Indexing:
|
|
185
191
|
"""
|
186
192
|
return self.data_providers
|
187
193
|
|
188
|
-
def get_loader_arguments(
|
194
|
+
def get_loader_arguments(
|
195
|
+
self,
|
196
|
+
id: str,
|
197
|
+
type: str = "file"
|
198
|
+
) -> Dict[str, Any]:
|
189
199
|
"""
|
190
200
|
Get keyword arguments for loader
|
191
201
|
|
@@ -267,8 +277,8 @@ class Indexing:
|
|
267
277
|
path: str,
|
268
278
|
force: bool = False,
|
269
279
|
silent: bool = False,
|
270
|
-
loader_kwargs:
|
271
|
-
) ->
|
280
|
+
loader_kwargs: Optional[Dict[str, Any]] = None,
|
281
|
+
) -> List[Document]:
|
272
282
|
"""
|
273
283
|
Get documents from path using data loaders
|
274
284
|
|
@@ -330,7 +340,11 @@ class Indexing:
|
|
330
340
|
self.window.core.idx.metadata.append_file_metadata(documents, path)
|
331
341
|
return documents
|
332
342
|
|
333
|
-
def read_text_content(
|
343
|
+
def read_text_content(
|
344
|
+
self,
|
345
|
+
path: str,
|
346
|
+
loader_kwargs: Optional[Dict[str, Any]] = None
|
347
|
+
) -> str:
|
334
348
|
"""
|
335
349
|
Get content from file using loaders
|
336
350
|
|
@@ -353,7 +367,7 @@ class Indexing:
|
|
353
367
|
self,
|
354
368
|
url: str,
|
355
369
|
type: str = "webpage",
|
356
|
-
extra_args:
|
370
|
+
extra_args: Optional[Dict[str, Any]] = None
|
357
371
|
) -> str:
|
358
372
|
"""
|
359
373
|
Get content from external resource
|
@@ -373,8 +387,8 @@ class Indexing:
|
|
373
387
|
self,
|
374
388
|
url: str,
|
375
389
|
type: str = "webpage",
|
376
|
-
extra_args:
|
377
|
-
) ->
|
390
|
+
extra_args: Optional[Dict[str, Any]] = None,
|
391
|
+
) -> List[Document]:
|
378
392
|
"""
|
379
393
|
Read data from external resource
|
380
394
|
|
@@ -423,11 +437,11 @@ class Indexing:
|
|
423
437
|
self,
|
424
438
|
idx: str,
|
425
439
|
index: BaseIndex,
|
426
|
-
path: str = None,
|
440
|
+
path: Optional[str] = None,
|
427
441
|
is_tmp: bool = False,
|
428
|
-
replace: bool = None,
|
429
|
-
recursive: bool = None
|
430
|
-
) ->
|
442
|
+
replace: Optional[bool] = None,
|
443
|
+
recursive: Optional[bool] = None
|
444
|
+
) -> Tuple[dict, list]:
|
431
445
|
"""
|
432
446
|
Index all files in directory
|
433
447
|
|
@@ -494,10 +508,10 @@ class Indexing:
|
|
494
508
|
self,
|
495
509
|
idx: str,
|
496
510
|
index: BaseIndex,
|
497
|
-
path: str = None,
|
511
|
+
path: Optional[str] = None,
|
498
512
|
is_tmp: bool = False,
|
499
|
-
replace: bool = None
|
500
|
-
) ->
|
513
|
+
replace: Optional[bool] = None
|
514
|
+
) -> Tuple[dict, list]:
|
501
515
|
"""
|
502
516
|
Index all files in directory and subdirectories recursively.
|
503
517
|
|
@@ -688,7 +702,13 @@ class Indexing:
|
|
688
702
|
documents.append(doc)
|
689
703
|
return documents
|
690
704
|
|
691
|
-
def index_db_by_meta_id(
|
705
|
+
def index_db_by_meta_id(
|
706
|
+
self,
|
707
|
+
idx: str,
|
708
|
+
index: BaseIndex,
|
709
|
+
id: int = 0,
|
710
|
+
from_ts: int = 0
|
711
|
+
) -> Tuple[int, List[str]]:
|
692
712
|
"""
|
693
713
|
Index data from database by meta id
|
694
714
|
|
@@ -724,7 +744,12 @@ class Indexing:
|
|
724
744
|
self.window.core.debug.log(e)
|
725
745
|
return n, errors
|
726
746
|
|
727
|
-
def index_db_from_updated_ts(
|
747
|
+
def index_db_from_updated_ts(
|
748
|
+
self,
|
749
|
+
idx: str,
|
750
|
+
index: BaseIndex,
|
751
|
+
from_ts: int = 0
|
752
|
+
) -> Tuple[int, List[str]]:
|
728
753
|
"""
|
729
754
|
Index data from database from timestamp
|
730
755
|
|
@@ -752,10 +777,10 @@ class Indexing:
|
|
752
777
|
index: BaseIndex,
|
753
778
|
url: str,
|
754
779
|
type="webpage",
|
755
|
-
extra_args:
|
780
|
+
extra_args: Optional[Dict[str, Any]] = None,
|
756
781
|
is_tmp: bool = False,
|
757
|
-
replace: bool = None
|
758
|
-
) ->
|
782
|
+
replace: Optional[bool] = None
|
783
|
+
) -> Tuple[int, List[str]]:
|
759
784
|
"""
|
760
785
|
Index data from external (remote) resource
|
761
786
|
|
@@ -838,9 +863,9 @@ class Indexing:
|
|
838
863
|
index: BaseIndex,
|
839
864
|
urls: list,
|
840
865
|
type="webpage",
|
841
|
-
extra_args:
|
866
|
+
extra_args: Optional[Dict[str, Any]] = None,
|
842
867
|
is_tmp: bool = False
|
843
|
-
) ->
|
868
|
+
) -> Tuple[int, List[str]]:
|
844
869
|
"""
|
845
870
|
Index data from URLs
|
846
871
|
|
@@ -878,7 +903,12 @@ class Indexing:
|
|
878
903
|
errors.extend(errs)
|
879
904
|
return n, errors
|
880
905
|
|
881
|
-
def remove_old_meta_id(
|
906
|
+
def remove_old_meta_id(
|
907
|
+
self,
|
908
|
+
idx: str,
|
909
|
+
id: int = 0,
|
910
|
+
force: bool = False
|
911
|
+
) -> bool:
|
882
912
|
"""
|
883
913
|
Remove old meta id from index
|
884
914
|
|
@@ -906,7 +936,12 @@ class Indexing:
|
|
906
936
|
return True
|
907
937
|
return False
|
908
938
|
|
909
|
-
def remove_old_file(
|
939
|
+
def remove_old_file(
|
940
|
+
self,
|
941
|
+
idx: str,
|
942
|
+
file_id: str,
|
943
|
+
force: bool = False
|
944
|
+
) -> bool:
|
910
945
|
"""
|
911
946
|
Remove old file from index
|
912
947
|
|
@@ -934,7 +969,13 @@ class Indexing:
|
|
934
969
|
return True
|
935
970
|
return False
|
936
971
|
|
937
|
-
def remove_old_external(
|
972
|
+
def remove_old_external(
|
973
|
+
self,
|
974
|
+
idx: str,
|
975
|
+
content: str,
|
976
|
+
type: str,
|
977
|
+
force: bool = False
|
978
|
+
) -> bool:
|
938
979
|
"""
|
939
980
|
Remove old file from index
|
940
981
|
|
@@ -963,7 +1004,11 @@ class Indexing:
|
|
963
1004
|
return True
|
964
1005
|
return False
|
965
1006
|
|
966
|
-
def index_document(
|
1007
|
+
def index_document(
|
1008
|
+
self,
|
1009
|
+
index: BaseIndex,
|
1010
|
+
doc: Document
|
1011
|
+
):
|
967
1012
|
"""
|
968
1013
|
Index document
|
969
1014
|
|
@@ -983,8 +1028,8 @@ class Indexing:
|
|
983
1028
|
self,
|
984
1029
|
file_path: str,
|
985
1030
|
index_path: str,
|
986
|
-
model: ModelItem = None,
|
987
|
-
documents:
|
1031
|
+
model: Optional[ModelItem] = None,
|
1032
|
+
documents: Optional[List[Document]] = None,
|
988
1033
|
) -> list:
|
989
1034
|
"""
|
990
1035
|
Index context attachment
|
@@ -1021,8 +1066,8 @@ class Indexing:
|
|
1021
1066
|
self,
|
1022
1067
|
url: str,
|
1023
1068
|
index_path: str,
|
1024
|
-
model: ModelItem = None,
|
1025
|
-
documents:
|
1069
|
+
model: Optional[ModelItem] = None,
|
1070
|
+
documents: Optional[List[Document]] = None,
|
1026
1071
|
) -> list:
|
1027
1072
|
"""
|
1028
1073
|
Index context attachment
|
@@ -1077,7 +1122,11 @@ class Indexing:
|
|
1077
1122
|
print("Selected web data loader: {}".format(type))
|
1078
1123
|
return type
|
1079
1124
|
|
1080
|
-
def remove_attachment(
|
1125
|
+
def remove_attachment(
|
1126
|
+
self,
|
1127
|
+
index_path: str,
|
1128
|
+
doc_id: str
|
1129
|
+
) -> bool:
|
1081
1130
|
"""
|
1082
1131
|
Remove document from index
|
1083
1132
|
|
pygpt_net/core/idx/llm.py
CHANGED
@@ -6,10 +6,11 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 08:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import os.path
|
13
|
+
from typing import Optional, Union
|
13
14
|
|
14
15
|
from llama_index.core.llms.llm import BaseLLM
|
15
16
|
from llama_index.core.multi_modal_llms import MultiModalLLM
|
@@ -41,7 +42,11 @@ class Llm:
|
|
41
42
|
os.environ['OPENAI_API_BASE'] = str(self.window.core.config.get('api_endpoint'))
|
42
43
|
os.environ['OPENAI_ORGANIZATION'] = str(self.window.core.config.get('organization_key'))
|
43
44
|
|
44
|
-
def get(
|
45
|
+
def get(
|
46
|
+
self,
|
47
|
+
model: Optional[ModelItem] = None,
|
48
|
+
multimodal: bool = False
|
49
|
+
) -> Union[BaseLLM, MultiModalLLM]:
|
45
50
|
"""
|
46
51
|
Get LLM provider
|
47
52
|
|
@@ -117,7 +122,10 @@ class Llm:
|
|
117
122
|
config=args,
|
118
123
|
)
|
119
124
|
|
120
|
-
def get_service_context(
|
125
|
+
def get_service_context(
|
126
|
+
self,
|
127
|
+
model: Optional[ModelItem] = None
|
128
|
+
) -> ServiceContext:
|
121
129
|
"""
|
122
130
|
Get service context + embeddings provider
|
123
131
|
|
pygpt_net/core/idx/metadata.py
CHANGED
@@ -6,11 +6,12 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 08:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
13
13
|
import os.path
|
14
|
+
from typing import List, Dict, Any
|
14
15
|
|
15
16
|
from llama_index.core.schema import Document
|
16
17
|
|
@@ -24,7 +25,11 @@ class Metadata:
|
|
24
25
|
"""
|
25
26
|
self.window = window
|
26
27
|
|
27
|
-
def append_file_metadata(
|
28
|
+
def append_file_metadata(
|
29
|
+
self,
|
30
|
+
docs: List[Document],
|
31
|
+
path: str
|
32
|
+
):
|
28
33
|
"""
|
29
34
|
Append custom meta to documents (file)
|
30
35
|
|
@@ -81,7 +86,12 @@ class Metadata:
|
|
81
86
|
self.window.core.debug.log(e)
|
82
87
|
doc.metadata = meta
|
83
88
|
|
84
|
-
def append_web_metadata(
|
89
|
+
def append_web_metadata(
|
90
|
+
self,
|
91
|
+
docs: List[Document],
|
92
|
+
type: str,
|
93
|
+
args: Dict[str, Any]
|
94
|
+
):
|
85
95
|
"""
|
86
96
|
Append custom meta to documents (web)
|
87
97
|
|
pygpt_net/core/idx/types/ctx.py
CHANGED
@@ -6,9 +6,12 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 00:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
|
+
from typing import Optional
|
13
|
+
|
14
|
+
|
12
15
|
class Ctx:
|
13
16
|
def __init__(self, window=None, provider=None):
|
14
17
|
"""
|
@@ -43,7 +46,12 @@ class Ctx:
|
|
43
46
|
doc_id=doc_id,
|
44
47
|
)
|
45
48
|
|
46
|
-
def exists(
|
49
|
+
def exists(
|
50
|
+
self,
|
51
|
+
store_id: str,
|
52
|
+
idx: str,
|
53
|
+
meta_id: int
|
54
|
+
) -> bool:
|
47
55
|
"""
|
48
56
|
Check if ctx meta is indexed
|
49
57
|
|
@@ -58,7 +66,12 @@ class Ctx:
|
|
58
66
|
meta_id=meta_id,
|
59
67
|
)
|
60
68
|
|
61
|
-
def get_doc_id(
|
69
|
+
def get_doc_id(
|
70
|
+
self,
|
71
|
+
store_id: str,
|
72
|
+
idx: str,
|
73
|
+
meta_id: int
|
74
|
+
) -> str:
|
62
75
|
"""
|
63
76
|
Get indexed document id by meta id
|
64
77
|
|
@@ -73,7 +86,11 @@ class Ctx:
|
|
73
86
|
meta_id=meta_id,
|
74
87
|
)
|
75
88
|
|
76
|
-
def update(
|
89
|
+
def update(
|
90
|
+
self,
|
91
|
+
meta_id: int,
|
92
|
+
doc_id: str
|
93
|
+
) -> bool:
|
77
94
|
"""
|
78
95
|
Update timestamp of indexed ctx meta
|
79
96
|
|
@@ -86,7 +103,12 @@ class Ctx:
|
|
86
103
|
doc_id=doc_id,
|
87
104
|
)
|
88
105
|
|
89
|
-
def remove(
|
106
|
+
def remove(
|
107
|
+
self,
|
108
|
+
store_id: str,
|
109
|
+
idx: str,
|
110
|
+
meta_id: str
|
111
|
+
):
|
90
112
|
"""
|
91
113
|
Remove document from index db
|
92
114
|
|
@@ -100,7 +122,11 @@ class Ctx:
|
|
100
122
|
meta_id=meta_id,
|
101
123
|
)
|
102
124
|
|
103
|
-
def truncate(
|
125
|
+
def truncate(
|
126
|
+
self,
|
127
|
+
store_id: Optional[str] = None,
|
128
|
+
idx: Optional[str] = None
|
129
|
+
):
|
104
130
|
"""
|
105
131
|
Truncate ctx meta from index db
|
106
132
|
|
@@ -6,11 +6,12 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 00:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
13
13
|
import time
|
14
|
+
from typing import Optional
|
14
15
|
|
15
16
|
|
16
17
|
class External:
|
@@ -54,7 +55,13 @@ class External:
|
|
54
55
|
data=data,
|
55
56
|
)
|
56
57
|
|
57
|
-
def get_doc_id(
|
58
|
+
def get_doc_id(
|
59
|
+
self,
|
60
|
+
store_id: str,
|
61
|
+
idx: str,
|
62
|
+
content: str,
|
63
|
+
type: str
|
64
|
+
) -> str:
|
58
65
|
"""
|
59
66
|
Get indexed document id by content
|
60
67
|
|
@@ -71,7 +78,13 @@ class External:
|
|
71
78
|
type=type,
|
72
79
|
)
|
73
80
|
|
74
|
-
def exists(
|
81
|
+
def exists(
|
82
|
+
self,
|
83
|
+
store_id: str,
|
84
|
+
idx: str,
|
85
|
+
content: str,
|
86
|
+
type: str
|
87
|
+
) -> bool:
|
75
88
|
"""
|
76
89
|
Check if content is indexed
|
77
90
|
|
@@ -88,7 +101,13 @@ class External:
|
|
88
101
|
type=type,
|
89
102
|
)
|
90
103
|
|
91
|
-
def update(
|
104
|
+
def update(
|
105
|
+
self,
|
106
|
+
content: str,
|
107
|
+
type: str,
|
108
|
+
doc_id: str,
|
109
|
+
ts: int
|
110
|
+
) -> bool:
|
92
111
|
"""
|
93
112
|
Update timestamp of indexed external content
|
94
113
|
|
@@ -105,7 +124,12 @@ class External:
|
|
105
124
|
ts=ts,
|
106
125
|
)
|
107
126
|
|
108
|
-
def remove(
|
127
|
+
def remove(
|
128
|
+
self,
|
129
|
+
store_id: str,
|
130
|
+
idx: str,
|
131
|
+
doc_id: str
|
132
|
+
):
|
109
133
|
"""
|
110
134
|
Remove document from index
|
111
135
|
|
@@ -119,7 +143,11 @@ class External:
|
|
119
143
|
doc_id=doc_id,
|
120
144
|
)
|
121
145
|
|
122
|
-
def truncate(
|
146
|
+
def truncate(
|
147
|
+
self,
|
148
|
+
store_id: Optional[str] = None,
|
149
|
+
idx: Optional[str] = None
|
150
|
+
):
|
123
151
|
"""
|
124
152
|
Truncate external data from index
|
125
153
|
|
@@ -131,7 +159,13 @@ class External:
|
|
131
159
|
idx=idx,
|
132
160
|
)
|
133
161
|
|
134
|
-
def set_indexed(
|
162
|
+
def set_indexed(
|
163
|
+
self,
|
164
|
+
content: str,
|
165
|
+
type: str,
|
166
|
+
idx: str,
|
167
|
+
doc_id: str
|
168
|
+
) -> bool:
|
135
169
|
"""
|
136
170
|
Set external content as indexed
|
137
171
|
|
@@ -6,11 +6,12 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 00:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import datetime
|
13
13
|
import os.path
|
14
|
+
from typing import Optional
|
14
15
|
|
15
16
|
|
16
17
|
class Files:
|
@@ -67,7 +68,12 @@ class Files:
|
|
67
68
|
path = path.replace("\\", "/").strip(r'\/')
|
68
69
|
return path
|
69
70
|
|
70
|
-
def get_doc_id(
|
71
|
+
def get_doc_id(
|
72
|
+
self,
|
73
|
+
store_id: str,
|
74
|
+
idx: str,
|
75
|
+
file_id: str
|
76
|
+
) -> str:
|
71
77
|
"""
|
72
78
|
Get indexed document id by file
|
73
79
|
|
@@ -82,7 +88,12 @@ class Files:
|
|
82
88
|
file_id=file_id,
|
83
89
|
)
|
84
90
|
|
85
|
-
def exists(
|
91
|
+
def exists(
|
92
|
+
self,
|
93
|
+
store_id: str,
|
94
|
+
idx: str,
|
95
|
+
file_id: str
|
96
|
+
) -> bool:
|
86
97
|
"""
|
87
98
|
Check if file is indexed
|
88
99
|
|
@@ -97,7 +108,12 @@ class Files:
|
|
97
108
|
file_id=file_id,
|
98
109
|
)
|
99
110
|
|
100
|
-
def update(
|
111
|
+
def update(
|
112
|
+
self,
|
113
|
+
id: int,
|
114
|
+
doc_id: str,
|
115
|
+
ts: int
|
116
|
+
) -> bool:
|
101
117
|
"""
|
102
118
|
Update timestamp of indexed file
|
103
119
|
|
@@ -112,7 +128,12 @@ class Files:
|
|
112
128
|
ts=ts,
|
113
129
|
)
|
114
130
|
|
115
|
-
def remove(
|
131
|
+
def remove(
|
132
|
+
self,
|
133
|
+
store_id: str,
|
134
|
+
idx: str,
|
135
|
+
doc_id: str
|
136
|
+
):
|
116
137
|
"""
|
117
138
|
Remove document from index
|
118
139
|
|
@@ -126,7 +147,11 @@ class Files:
|
|
126
147
|
doc_id=doc_id,
|
127
148
|
)
|
128
149
|
|
129
|
-
def truncate(
|
150
|
+
def truncate(
|
151
|
+
self,
|
152
|
+
store_id: Optional[str] = None,
|
153
|
+
idx: Optional[str] = None
|
154
|
+
):
|
130
155
|
"""
|
131
156
|
Truncate files from index
|
132
157
|
|
pygpt_net/core/image.py
CHANGED
@@ -6,10 +6,11 @@
|
|
6
6
|
# GitHub: https://github.com/szczyglis-dev/py-gpt #
|
7
7
|
# MIT License #
|
8
8
|
# Created By : Marcin Szczygliński #
|
9
|
-
# Updated Date: 2024.
|
9
|
+
# Updated Date: 2024.12.14 08:00:00 #
|
10
10
|
# ================================================== #
|
11
11
|
|
12
12
|
import os
|
13
|
+
from typing import List
|
13
14
|
|
14
15
|
from PySide6.QtCore import Slot, QObject
|
15
16
|
|
@@ -34,7 +35,12 @@ class Image(QObject):
|
|
34
35
|
os.makedirs(img_dir, exist_ok=True)
|
35
36
|
|
36
37
|
@Slot(object, list, str)
|
37
|
-
def handle_finished(
|
38
|
+
def handle_finished(
|
39
|
+
self,
|
40
|
+
ctx: CtxItem,
|
41
|
+
paths: List[str],
|
42
|
+
prompt: str
|
43
|
+
):
|
38
44
|
"""
|
39
45
|
Handle finished image generation
|
40
46
|
|
@@ -45,7 +51,12 @@ class Image(QObject):
|
|
45
51
|
self.window.controller.chat.image.handle_response(ctx, paths, prompt)
|
46
52
|
|
47
53
|
@Slot(object, list, str)
|
48
|
-
def handle_finished_inline(
|
54
|
+
def handle_finished_inline(
|
55
|
+
self,
|
56
|
+
ctx: CtxItem,
|
57
|
+
paths: List[str],
|
58
|
+
prompt: str
|
59
|
+
):
|
49
60
|
"""
|
50
61
|
Handle finished image generation
|
51
62
|
|
@@ -86,7 +97,7 @@ class Image(QObject):
|
|
86
97
|
self.window.update_status(msg)
|
87
98
|
self.window.core.debug.log(msg)
|
88
99
|
|
89
|
-
def save_image(self, path: str, image:
|
100
|
+
def save_image(self, path: str, image: bytes) -> bool:
|
90
101
|
"""
|
91
102
|
Save image to file
|
92
103
|
|