sunholo 0.79.4__py3-none-any.whl → 0.79.5__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunholo/chunker/splitter.py +4 -3
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/METADATA +2 -2
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/RECORD +7 -7
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/LICENSE.txt +0 -0
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/WHEEL +0 -0
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/entry_points.txt +0 -0
- {sunholo-0.79.4.dist-info → sunholo-0.79.5.dist-info}/top_level.txt +0 -0
sunholo/chunker/splitter.py
CHANGED
|
@@ -113,8 +113,9 @@ def choose_splitter(extension: str, chunk_size: int=1024, chunk_overlap:int=200,
|
|
|
113
113
|
|
|
114
114
|
if vector_name:
|
|
115
115
|
# check if there is a chunking configuration
|
|
116
|
-
from ..utils import
|
|
117
|
-
|
|
116
|
+
from ..utils import ConfigManager
|
|
117
|
+
config = ConfigManager(vector_name)
|
|
118
|
+
chunk_config = config.vacConfig("chunker")
|
|
118
119
|
if chunk_config:
|
|
119
120
|
if chunk_config.get("type") == "semantic":
|
|
120
121
|
embedding_str = chunk_config.get("llm")
|
|
@@ -124,7 +125,7 @@ def choose_splitter(extension: str, chunk_size: int=1024, chunk_overlap:int=200,
|
|
|
124
125
|
log.info(f"Semantic chunking for {vector_name}")
|
|
125
126
|
from langchain_experimental.text_splitter import SemanticChunker
|
|
126
127
|
from ..components import pick_embedding
|
|
127
|
-
embeddings = pick_embedding(embedding_str)
|
|
128
|
+
embeddings = pick_embedding(embedding_str, config=config)
|
|
128
129
|
semantic_splitter = SemanticChunker(
|
|
129
130
|
embeddings, breakpoint_threshold_type="percentile"
|
|
130
131
|
)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sunholo
|
|
3
|
-
Version: 0.79.
|
|
3
|
+
Version: 0.79.5
|
|
4
4
|
Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
|
|
5
5
|
Home-page: https://github.com/sunholo-data/sunholo-py
|
|
6
|
-
Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.
|
|
6
|
+
Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.79.5.tar.gz
|
|
7
7
|
Author: Holosun ApS
|
|
8
8
|
Author-email: multivac@sunholo.com
|
|
9
9
|
License: Apache License, Version 2.0
|
|
@@ -40,7 +40,7 @@ sunholo/chunker/pdfs.py,sha256=njDPop751GMHi3cOwIKd2Yct-_lWR2gqcB7WykfHphs,2480
|
|
|
40
40
|
sunholo/chunker/process_chunker_data.py,sha256=OnMvXHRv3rGpFsU50FyUNkNIwC1D8TkhaWWbn72yQss,3523
|
|
41
41
|
sunholo/chunker/publish.py,sha256=AX5u-fcyDytED67IfizMzvOMcYPXEo6XBJvyk_7maK8,2939
|
|
42
42
|
sunholo/chunker/pubsub.py,sha256=48bhuAcszN7LGe3-ksPSLHHhq0uKxiXOrizck5qpcP0,1012
|
|
43
|
-
sunholo/chunker/splitter.py,sha256=
|
|
43
|
+
sunholo/chunker/splitter.py,sha256=QLAEsJOpEYFZr9-UGZUuAlNVyjfCWb8jvzCHg0rVShE,6751
|
|
44
44
|
sunholo/cli/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
45
45
|
sunholo/cli/chat_vac.py,sha256=UNLzZYAejjEvMR0EjpxIHXyvUpBDoqThohzzFL_m3Yg,23026
|
|
46
46
|
sunholo/cli/cli.py,sha256=yuY7SLFiYDUKqJDOXy7jL1l6P0UVPMuAZK9bXXTG8ck,3939
|
|
@@ -132,9 +132,9 @@ sunholo/vertex/init.py,sha256=1OQwcPBKZYBTDPdyU7IM4X4OmiXLdsNV30C-fee2scQ,2875
|
|
|
132
132
|
sunholo/vertex/memory_tools.py,sha256=pgSahVDh7GPEulu3nl-w0jb5lTClb4TCnVxPnMokNZY,7533
|
|
133
133
|
sunholo/vertex/safety.py,sha256=S9PgQT1O_BQAkcqauWncRJaydiP8Q_Jzmu9gxYfy1VA,2482
|
|
134
134
|
sunholo/vertex/type_dict_to_json.py,sha256=uTzL4o9tJRao4u-gJOFcACgWGkBOtqACmb6ihvCErL8,4694
|
|
135
|
-
sunholo-0.79.
|
|
136
|
-
sunholo-0.79.
|
|
137
|
-
sunholo-0.79.
|
|
138
|
-
sunholo-0.79.
|
|
139
|
-
sunholo-0.79.
|
|
140
|
-
sunholo-0.79.
|
|
135
|
+
sunholo-0.79.5.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
|
|
136
|
+
sunholo-0.79.5.dist-info/METADATA,sha256=B4e56x_LDOAPMTDXey40Sa4P0P9j69ZqL2pZUVjPaj8,7348
|
|
137
|
+
sunholo-0.79.5.dist-info/WHEEL,sha256=R0nc6qTxuoLk7ShA2_Y-UWkN8ZdfDBG2B6Eqpz2WXbs,91
|
|
138
|
+
sunholo-0.79.5.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
|
|
139
|
+
sunholo-0.79.5.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
|
|
140
|
+
sunholo-0.79.5.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|