raglab_bak 0.1.3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- raglab_bak/Dockerfile +12 -0
- raglab_bak/__init__.py +3 -0
- raglab_bak/audio.py +18 -0
- raglab_bak/fiddle_manager/service.py +32 -0
- raglab_bak/mermaid/prompt_editor.md +13 -0
- raglab_bak/raglab2.py +174 -0
- raglab_bak/retrieval/VectorDB.py +163 -0
- raglab_bak/retrieval/__init__.py +0 -0
- raglab_bak/retrieval/chroma_client.py +31 -0
- raglab_bak/retrieval/lib_alexis.py +429 -0
- raglab_bak/retrieval/segmentation_lib.py +262 -0
- raglab_bak/stores/__init__.py +1 -0
- raglab_bak/stores/function_store.py +71 -0
- raglab_bak/stores/repos.py +39 -0
- raglab_bak/stores/simple_stores.py +64 -0
- raglab_bak/stores/sql_app_stores.py +67 -0
- raglab_bak/stores/stores_util.py +128 -0
- raglab_bak/stores/user_store.py +9 -0
- raglab_bak/suggestion_engine/__init__.py +0 -0
- raglab_bak/suggestion_engine/suggestion_example.ipynb +317 -0
- raglab_bak/suggestion_engine/suggestion_lib.py +156 -0
- raglab_bak/templates/chat.html +131 -0
- raglab_bak/templates/gate.html +31 -0
- raglab_bak/templates/login.html +37 -0
- raglab_bak/templates/raglab_home.html +33 -0
- raglab_bak/templates/raglab_query.html +69 -0
- raglab_bak/templates/raglab_query_result.html +42 -0
- raglab_bak/tests/test.db +0 -0
- raglab_bak/tests/test_app.py +65 -0
- raglab_bak/tests/test_repo.py +0 -0
- raglab_bak/tests/test_stores_util.py +60 -0
- raglab_bak/transformations.py +25 -0
- raglab_bak/util.py +217 -0
- raglab_bak/web_services/__init__.py +1 -0
- raglab_bak/web_services/mockups.py +143 -0
- raglab_bak/web_services/prompt_pilote.py +73 -0
- raglab_bak/web_services/prompt_templates_dde.py +110 -0
- raglab_bak/web_services/store_access.py +47 -0
- raglab_bak/wips/__init__.py +1 -0
- raglab_bak/wips/raglab_wips.ipynb +299 -0
- raglab_bak/wips/vd_facade_1/__init__.py +10 -0
- raglab_bak/wips/vd_facade_1/components.py +88 -0
- raglab_bak/wips/vd_facade_1/config.py +31 -0
- raglab_bak/wips/vd_facade_1/dev_instructions_1.md +843 -0
- raglab_bak/wips/vd_facade_1/dev_notes.md +6 -0
- raglab_bak/wips/vd_facade_1/interface.py +78 -0
- raglab_bak/wips/vd_facade_1/langchain_integration.py +329 -0
- raglab_bak/wips/vd_facade_1/registry.py +71 -0
- raglab_bak/wips/vd_facade_1/tests/__init__.py +1 -0
- raglab_bak/wips/vd_facade_1/tests/test_basic.py +23 -0
- raglab_bak/wips/vd_facade_1/tests/test_integration.py +207 -0
- raglab_bak/wips/vd_facade_1/tests/test_qdrant_integration.py +62 -0
- raglab_bak/wips/vd_facade_1/types.py +73 -0
- raglab_bak/wips/vectordb_discovery.py +490 -0
- raglab_bak/wips/vectordb_manager.py +1162 -0
- raglab_bak-0.1.3.dist-info/METADATA +71 -0
- raglab_bak-0.1.3.dist-info/RECORD +59 -0
- raglab_bak-0.1.3.dist-info/WHEEL +4 -0
- raglab_bak-0.1.3.dist-info/licenses/LICENSE +21 -0
raglab_bak/Dockerfile
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
FROM python:3.9
|
|
2
|
+
|
|
3
|
+
WORKDIR /app
|
|
4
|
+
|
|
5
|
+
COPY . /app
|
|
6
|
+
|
|
7
|
+
COPY .env /app
|
|
8
|
+
COPY test.db /app
|
|
9
|
+
|
|
10
|
+
RUN pip install fastapi gpt4all openai langchain chromadb lkj config2py dill dol tiktoken oa ju psycopg2
|
|
11
|
+
RUN pip install git+https://github.com/i2mint/sqldol
|
|
12
|
+
CMD ["python", "app.py"]
|
raglab_bak/__init__.py
ADDED
raglab_bak/audio.py
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
"""Audio utilities for RAG ingestion (e.g. silence removal)."""
|
|
2
|
+
|
|
3
|
+
from pydub import AudioSegment, silence
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
def remove_silence_from_audio(audio_file_path, silence_threshold=-50.0, chunk_size=10):
|
|
7
|
+
audio = AudioSegment.from_file(audio_file_path)
|
|
8
|
+
|
|
9
|
+
non_silent_parts = silence.detect_nonsilent(
|
|
10
|
+
audio, min_silence_len=chunk_size, silence_thresh=silence_threshold
|
|
11
|
+
)
|
|
12
|
+
|
|
13
|
+
processed_audio = AudioSegment.silent(duration=0)
|
|
14
|
+
|
|
15
|
+
for start, end in non_silent_parts:
|
|
16
|
+
processed_audio += audio[start:end]
|
|
17
|
+
|
|
18
|
+
return processed_audio
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
"""Fiddle manager service: CRUD over per-user named objects, served over HTTP via py2http."""
|
|
2
|
+
|
|
3
|
+
from stores.repos import RepoJson
|
|
4
|
+
from py2http import mk_flat, run_app
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
class FiddleManager:
|
|
8
|
+
def __init__(self):
|
|
9
|
+
self.repo = RepoJson()
|
|
10
|
+
|
|
11
|
+
def create(self, user_id: str, name: str):
|
|
12
|
+
print("Creating object")
|
|
13
|
+
self.repo.create(user_id + name, "")
|
|
14
|
+
|
|
15
|
+
def get(self, user_id: str, name: str) -> str:
|
|
16
|
+
return self.repo.get(user_id + name)
|
|
17
|
+
|
|
18
|
+
def update(self, user_id: str, name: str, value: str):
|
|
19
|
+
return self.repo.update(user_id + name, value)
|
|
20
|
+
|
|
21
|
+
def delete(self, user_id: str, name: str):
|
|
22
|
+
return self.repo.delete(user_id + name)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def run_fiddle_manager():
|
|
26
|
+
create = mk_flat(FiddleManager, FiddleManager.create, func_name="create")
|
|
27
|
+
get = mk_flat(FiddleManager, FiddleManager.get, func_name="get")
|
|
28
|
+
update = mk_flat(FiddleManager, FiddleManager.update, func_name="update")
|
|
29
|
+
delete = mk_flat(FiddleManager, FiddleManager.delete, func_name="delete")
|
|
30
|
+
fns = [create, get, update, delete]
|
|
31
|
+
|
|
32
|
+
run_app(fns, publish_openapi=True, publish_swagger=True)
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
```mermaid
|
|
2
|
+
sequenceDiagram
|
|
3
|
+
participant Frontend
|
|
4
|
+
participant Backend
|
|
5
|
+
Frontend->>Backend: GET /prompt
|
|
6
|
+
Backend-->>Frontend: JSON { [{name:str}] } (liste des templates)
|
|
7
|
+
Frontend->>Backend: GET /prompt/editor?name (récupérer le template + les forms)
|
|
8
|
+
Backend-->>Frontend: JSON {template:str, rjsf_ui:str} (template + ui)
|
|
9
|
+
Frontend->>Backend: POST /prompt/editor JSON (poster le template pour le sauvegarder)
|
|
10
|
+
Backend-->>Frontend: JSON {template, rjsf_ui} (template + rjsf_ui mis à jour)
|
|
11
|
+
Frontend->>Backend: GET /prompt/editor/ask?query
|
|
12
|
+
Backend-->>Frontend: JSON { answer:str }
|
|
13
|
+
```
|
raglab_bak/raglab2.py
ADDED
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Taking the use case implemented by example:
|
|
3
|
+
https://python.langchain.com/docs/expression_language/cookbook/retrieval
|
|
4
|
+
I created a class that encapsulates the logic of the example, and added some
|
|
5
|
+
features to it. The class is called Raglab2.
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
Example of use:
|
|
9
|
+
|
|
10
|
+
```python
|
|
11
|
+
from srag.tw.raglab2 import *
|
|
12
|
+
raglab = raglab2('test')
|
|
13
|
+
|
|
14
|
+
text_store = {
|
|
15
|
+
"doc1": "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It is named after the engineer Gustave Eiffel, whose company designed and built the tower.",
|
|
16
|
+
"doc2": "The Great Wall of China is a series of fortifications made of stone, brick, tamped earth, wood, and other materials, generally built along an east-to-west line across the historical northern borders of China.",
|
|
17
|
+
"doc3": "The light bulb was invented by Thomas Edison in the 19th century. It was a groundbreaking invention that allowed for the widespread use of electric light in homes and businesses.",
|
|
18
|
+
"doc4": "Photosynthesis is a process used by plants, algae, and certain bacteria to convert light energy into chemical energy stored in glucose, which can be used to fuel the organism's activities."
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
# text_store = wrap_in_langchain_document(text_store)
|
|
22
|
+
|
|
23
|
+
raglab.update_with_mapping(text_store)
|
|
24
|
+
|
|
25
|
+
raglab.ask("Who invented the light bulb and what is the process plants use to convert light energy?")
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from typing import Optional, Union
|
|
31
|
+
from collections.abc import Callable, Mapping
|
|
32
|
+
from functools import cached_property, partial
|
|
33
|
+
import os
|
|
34
|
+
import chromadb
|
|
35
|
+
from chromadb.config import Settings
|
|
36
|
+
|
|
37
|
+
from dotenv import load_dotenv
|
|
38
|
+
|
|
39
|
+
from lkj import import_object, chunker, clog # pip install lkj
|
|
40
|
+
from config2py import get_configs_folder_for_app # pip install config2py
|
|
41
|
+
|
|
42
|
+
from langchain_core.embeddings import Embeddings
|
|
43
|
+
from langchain_core.language_models.llms import LLM
|
|
44
|
+
from langchain_core.output_parsers import StrOutputParser
|
|
45
|
+
from langchain_core.prompts import ChatPromptTemplate
|
|
46
|
+
from langchain_core.runnables import RunnableLambda, RunnablePassthrough
|
|
47
|
+
from langchain_core.vectorstores import VectorStore
|
|
48
|
+
|
|
49
|
+
from langchain_community.llms.gpt4all import GPT4All
|
|
50
|
+
from langchain_community.embeddings.gpt4all import GPT4AllEmbeddings
|
|
51
|
+
from langchain_community.vectorstores.chroma import Chroma
|
|
52
|
+
|
|
53
|
+
from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
|
|
54
|
+
from langchain.chains import LLMChain
|
|
55
|
+
from langchain.prompts import PromptTemplate
|
|
56
|
+
|
|
57
|
+
from stores.repos import RepoJson
|
|
58
|
+
from util import StoreAccess
|
|
59
|
+
|
|
60
|
+
segmenter = partial(chunker, chk_size=1000, include_tail=True)
|
|
61
|
+
|
|
62
|
+
DFLT_VECTORSTORES_PERSIST_ROOTDIR = get_configs_folder_for_app(
|
|
63
|
+
"srag", configs_name="vectorstores"
|
|
64
|
+
)
|
|
65
|
+
|
|
66
|
+
DFLT_PROMPT_TEMPLATE = """Answer the question based only on the following context:
|
|
67
|
+
{context}
|
|
68
|
+
|
|
69
|
+
Question: {question}
|
|
70
|
+
"""
|
|
71
|
+
|
|
72
|
+
DFLT_PROMPT = ChatPromptTemplate.from_template(DFLT_PROMPT_TEMPLATE)
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class Raglab2:
|
|
76
|
+
def __init__(
|
|
77
|
+
self,
|
|
78
|
+
vectorstore: VectorStore,
|
|
79
|
+
embedding: Embeddings,
|
|
80
|
+
prompt: PromptTemplate,
|
|
81
|
+
model: LLM, # orca-mini-3b-gguf2-q4_0 all-MiniLM-L6-v2-f16.gguf orca-mini-3b-gguf2-q4_0.gguf mistral-7b-openorca.Q4_0.gguf
|
|
82
|
+
):
|
|
83
|
+
self.vectorstore = vectorstore
|
|
84
|
+
self.embedding = embedding
|
|
85
|
+
self.prompt = prompt
|
|
86
|
+
self.model = model
|
|
87
|
+
|
|
88
|
+
def update_with_mapping(
|
|
89
|
+
self,
|
|
90
|
+
mapping: Mapping[str, str],
|
|
91
|
+
*,
|
|
92
|
+
chk_size: int = 1000,
|
|
93
|
+
verbose: bool = False,
|
|
94
|
+
# filt: Callable = None # to control update rules
|
|
95
|
+
):
|
|
96
|
+
|
|
97
|
+
ids, texts = map(list, zip(*mapping.items()))
|
|
98
|
+
self.vectorstore.from_texts(texts=texts, ids=ids, embedding=self.embedding)
|
|
99
|
+
|
|
100
|
+
# TODO: Make the segmented (batched) update of the vectorstore work
|
|
101
|
+
# --> Need to find the right vectorstore method for that (maybe update_documents?)
|
|
102
|
+
# TODO: Figure out how to only update vectorstores when "docs" have changed
|
|
103
|
+
# (editted, deleted, added, etc.)
|
|
104
|
+
# _segmenter = partial(segmenter, chk_size=chk_size)
|
|
105
|
+
# _clog = clog(verbose, log_func=lambda x: print(f"segment {i}"))
|
|
106
|
+
|
|
107
|
+
# for i, _items in enumerate(_segmenter(mapping.items())):
|
|
108
|
+
# _clog(i)
|
|
109
|
+
# ids, texts = map(list, zip(*_items))
|
|
110
|
+
# self.vectorstore.from_texts(texts=texts, ids=ids, embedding=self.embedding)
|
|
111
|
+
|
|
112
|
+
return self
|
|
113
|
+
|
|
114
|
+
@cached_property
|
|
115
|
+
def retriever(self):
|
|
116
|
+
return self.vectorstore.as_retriever()
|
|
117
|
+
|
|
118
|
+
@cached_property
|
|
119
|
+
def chain(self):
|
|
120
|
+
return (
|
|
121
|
+
{"context": self.retriever, "question": RunnablePassthrough()}
|
|
122
|
+
| self.prompt
|
|
123
|
+
| self.model
|
|
124
|
+
| StrOutputParser()
|
|
125
|
+
)
|
|
126
|
+
|
|
127
|
+
def ask(self, query: str):
|
|
128
|
+
return self.chain.invoke(query)
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
class RaglabSession:
|
|
132
|
+
def __init__(self, app: Raglab2):
|
|
133
|
+
self.raglab_app = app
|
|
134
|
+
|
|
135
|
+
def ask(self, query: str):
|
|
136
|
+
return self.raglab_app.ask(quer)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
class RaglabSessionBuilder:
|
|
140
|
+
def build(
|
|
141
|
+
user: str,
|
|
142
|
+
name: str,
|
|
143
|
+
vectorstore: VectorStore,
|
|
144
|
+
embedding: Embeddings,
|
|
145
|
+
prompt: PromptTemplate,
|
|
146
|
+
model: LLM, # orca-mini-3b-gguf2-q4_0 all-MiniLM-L6-v2-f16.gguf orca-mini-3b-gguf2-q4_0.gguf mistral-7b-openorca.Q4_0.gguf
|
|
147
|
+
) -> RaglabSession:
|
|
148
|
+
|
|
149
|
+
raglab = Raglab2(vectorstore, embedding, prompt, model)
|
|
150
|
+
session = RaglabSession(raglab)
|
|
151
|
+
session.raglab_app = raglab
|
|
152
|
+
|
|
153
|
+
return session
|
|
154
|
+
|
|
155
|
+
def restore(user: str, name: str) -> RaglabSession:
|
|
156
|
+
pass
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
from langchain.docstore.document import Document
|
|
160
|
+
from dol import wrap_kvs
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def _wrap_kv_in_langchain_document(k, v):
|
|
164
|
+
return Document(page_content=v, metadata={"key": k})
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def wrap_in_langchain_document(mapping: Mapping[str, str]):
|
|
168
|
+
return wrap_kvs(mapping, postget=_wrap_kv_in_langchain_document)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def object_resolver(module_dot_path, obj_name=None, *args, **kwargs):
|
|
172
|
+
if obj_name is None:
|
|
173
|
+
return partial(object_resolver, module_dot_path)
|
|
174
|
+
return import_object(".".join([module_dot_path, obj_name]))
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
"""VectorDB"""
|
|
2
|
+
|
|
3
|
+
# VectorDB class
|
|
4
|
+
from typing import Dict, Any
|
|
5
|
+
from collections.abc import Mapping, Iterable
|
|
6
|
+
from dataclasses import dataclass, field
|
|
7
|
+
from types import SimpleNamespace
|
|
8
|
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
|
9
|
+
from functools import cached_property
|
|
10
|
+
|
|
11
|
+
DocKey = str
|
|
12
|
+
DocValue = str
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
from typing import Tuple, List
|
|
16
|
+
from collections.abc import Mapping
|
|
17
|
+
|
|
18
|
+
SegmentKey = tuple[DocKey, int, int]
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def generate_split_keys(
|
|
22
|
+
docs: dict, text_splitter: RecursiveCharacterTextSplitter, metadatas: list[dict]
|
|
23
|
+
) -> list[tuple[str, int, int]]:
|
|
24
|
+
|
|
25
|
+
documents = text_splitter.create_documents(
|
|
26
|
+
list(docs.values()), metadatas=list({"document_name": k} for k in docs.keys())
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
return [
|
|
30
|
+
(
|
|
31
|
+
d.metadata["document_name"],
|
|
32
|
+
d.metadata["start_index"],
|
|
33
|
+
d.metadata["start_index"] + len(d.page_content),
|
|
34
|
+
)
|
|
35
|
+
for d in documents
|
|
36
|
+
]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class SegmentMapping:
|
|
40
|
+
"""A class to represent a mapping between segments and documents."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, docs: Mapping, segment_keys: list[SegmentKey]):
|
|
43
|
+
self.docs = docs
|
|
44
|
+
self.segment_keys = segment_keys
|
|
45
|
+
self.document_keys = list(docs.keys())
|
|
46
|
+
|
|
47
|
+
def __iter__(self):
|
|
48
|
+
yield from self.segment_keys
|
|
49
|
+
|
|
50
|
+
def __getitem__(self, key: SegmentKey):
|
|
51
|
+
if isinstance(key, str):
|
|
52
|
+
return self.docs[key]
|
|
53
|
+
elif isinstance(key, tuple):
|
|
54
|
+
doc_key, start_idx, end_idx = key
|
|
55
|
+
return self.docs[doc_key][start_idx:end_idx]
|
|
56
|
+
else:
|
|
57
|
+
raise TypeError("Key must be a string or a tuple")
|
|
58
|
+
|
|
59
|
+
def __setitem__(self, key: SegmentKey, value: str):
|
|
60
|
+
if isinstance(key, str):
|
|
61
|
+
self.docs[key] = value
|
|
62
|
+
return
|
|
63
|
+
else:
|
|
64
|
+
doc_key, start_idx, end_idx = key
|
|
65
|
+
self.segment_keys.append(key)
|
|
66
|
+
self.docs[doc_key] = (
|
|
67
|
+
self.docs.get(doc_key, "")[:start_idx]
|
|
68
|
+
+ value
|
|
69
|
+
+ self.docs.get(doc_key, "")[end_idx:]
|
|
70
|
+
)
|
|
71
|
+
|
|
72
|
+
def __add__(self, other):
|
|
73
|
+
"""Add two SegmentMapping objects together. This will concatenate the documents and segment keys."""
|
|
74
|
+
return SegmentMapping(
|
|
75
|
+
{**self.docs, **other.docs}, self.segment_keys + other.segment_keys
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
def __len__(self):
|
|
79
|
+
return len(self.segment_keys)
|
|
80
|
+
|
|
81
|
+
def __contains__(self, key: SegmentKey):
|
|
82
|
+
if isinstance(key, str):
|
|
83
|
+
return key in self.document_keys
|
|
84
|
+
elif isinstance(key, tuple):
|
|
85
|
+
return key in self.segment_keys
|
|
86
|
+
else:
|
|
87
|
+
raise TypeError("Key must be a string or a tuple")
|
|
88
|
+
|
|
89
|
+
def __repr__(self):
|
|
90
|
+
representation = ""
|
|
91
|
+
for key in self.segment_keys:
|
|
92
|
+
representation += str(key) + " : " + str(self.__getitem__(key)) + "\n"
|
|
93
|
+
return representation
|
|
94
|
+
|
|
95
|
+
def values(self):
|
|
96
|
+
for key in self.segment_keys:
|
|
97
|
+
yield self.__getitem__(key)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass
|
|
101
|
+
class ChunkDB:
|
|
102
|
+
"""Contains a mapping from document keys to document content. Keys can either be a tuple (doc_key, start_idx, end_idx) or a string doc_key.
|
|
103
|
+
The documents are split into segments and the segments are stored in a SegmentMapping object.
|
|
104
|
+
The SegmentMapping object is a mapping from segment keys to segments."""
|
|
105
|
+
|
|
106
|
+
docs: Mapping[DocKey, DocValue]
|
|
107
|
+
# kwargs: Dict[str, Any] = field(default_factory=dict)
|
|
108
|
+
chunk_size: int = 400
|
|
109
|
+
chunk_overlap: int = 100
|
|
110
|
+
|
|
111
|
+
@cached_property
|
|
112
|
+
def segments(self):
|
|
113
|
+
return self.mk_segment_store(self.docs)
|
|
114
|
+
|
|
115
|
+
@cached_property
|
|
116
|
+
def segment_store(self):
|
|
117
|
+
return SimpleNamespace(segment_keys=self.segments)
|
|
118
|
+
|
|
119
|
+
@cached_property
|
|
120
|
+
def document_store(self):
|
|
121
|
+
return SimpleNamespace(docs=self.docs)
|
|
122
|
+
|
|
123
|
+
@cached_property
|
|
124
|
+
def mall(self):
|
|
125
|
+
return SimpleNamespace(docs=self.docs, segment_keys=self.segments)
|
|
126
|
+
|
|
127
|
+
def add(self, new_documents: Mapping[DocKey, DocValue], when_exists: str = "raise"):
|
|
128
|
+
"""Add new documents to the database. When_exists can be "raise", "skip" or "overwrite.
|
|
129
|
+
New documents has to be a mapping with the document key as the key and the document content as the value.
|
|
130
|
+
"""
|
|
131
|
+
|
|
132
|
+
for new_doc_key in new_documents:
|
|
133
|
+
if new_doc_key in self.segments:
|
|
134
|
+
if when_exists == "skip":
|
|
135
|
+
continue
|
|
136
|
+
elif when_exists == "overwrite":
|
|
137
|
+
self.segments[new_doc_key] = new_documents[new_doc_key]
|
|
138
|
+
else:
|
|
139
|
+
raise ValueError(
|
|
140
|
+
f"Document with key {new_doc_key} already exists. Use 'when_exists' to handle this case."
|
|
141
|
+
)
|
|
142
|
+
else:
|
|
143
|
+
self.segments = self.segments + self.mk_segment_store(new_documents)
|
|
144
|
+
|
|
145
|
+
def mk_segment_store(self, docs):
|
|
146
|
+
"""Creates and returns a segment store from a dictionary of documents. Segment store is a mapping from segment keys to segments."""
|
|
147
|
+
text_splitter = RecursiveCharacterTextSplitter(
|
|
148
|
+
chunk_size=self.chunk_size,
|
|
149
|
+
chunk_overlap=min(self.chunk_overlap, self.chunk_size - 1),
|
|
150
|
+
length_function=len,
|
|
151
|
+
add_start_index=True, # enforce the start index
|
|
152
|
+
is_separator_regex=False,
|
|
153
|
+
)
|
|
154
|
+
segment_keys = generate_split_keys(docs, text_splitter, metadatas=[])
|
|
155
|
+
return SegmentMapping(docs, segment_keys)
|
|
156
|
+
|
|
157
|
+
def search(self, query: str, k: int = 1) -> Iterable[DocKey]:
|
|
158
|
+
# TODO : return (yields) the k most similar documents to the query
|
|
159
|
+
pass
|
|
160
|
+
|
|
161
|
+
def add_metadata(self, metadata: Mapping[DocKey, dict]):
|
|
162
|
+
# TODO : add metadata to the documents
|
|
163
|
+
pass
|
|
File without changes
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
"""Client wrapper for connecting to a remote Chroma vector database."""
|
|
2
|
+
|
|
3
|
+
import os
|
|
4
|
+
import chromadb
|
|
5
|
+
from chromadb.config import Settings
|
|
6
|
+
from dotenv import load_dotenv
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ChromaClient:
|
|
10
|
+
def __init__(self):
|
|
11
|
+
load_dotenv()
|
|
12
|
+
chroma_key = os.getenv("CHROMA_KEY")
|
|
13
|
+
self.client = chromadb.HttpClient(
|
|
14
|
+
host="149.202.47.109",
|
|
15
|
+
port="45000",
|
|
16
|
+
settings=Settings(
|
|
17
|
+
chroma_client_auth_provider="chromadb.auth.token.TokenAuthClientProvider",
|
|
18
|
+
chroma_client_auth_credentials=chroma_key,
|
|
19
|
+
),
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
def get_collections(self):
|
|
23
|
+
collections = self.client.list_collections()
|
|
24
|
+
return collections
|
|
25
|
+
|
|
26
|
+
def add_collection(self, name: str):
|
|
27
|
+
self.client.create_collection(name)
|
|
28
|
+
|
|
29
|
+
def query(self, dataset_name: str, query: str):
|
|
30
|
+
collection = self.client.get_collection(dataset_name)
|
|
31
|
+
return collection.query(query_texts=query)
|