raglab_bak 0.1.3__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (59) hide show
  1. raglab_bak/Dockerfile +12 -0
  2. raglab_bak/__init__.py +3 -0
  3. raglab_bak/audio.py +18 -0
  4. raglab_bak/fiddle_manager/service.py +32 -0
  5. raglab_bak/mermaid/prompt_editor.md +13 -0
  6. raglab_bak/raglab2.py +174 -0
  7. raglab_bak/retrieval/VectorDB.py +163 -0
  8. raglab_bak/retrieval/__init__.py +0 -0
  9. raglab_bak/retrieval/chroma_client.py +31 -0
  10. raglab_bak/retrieval/lib_alexis.py +429 -0
  11. raglab_bak/retrieval/segmentation_lib.py +262 -0
  12. raglab_bak/stores/__init__.py +1 -0
  13. raglab_bak/stores/function_store.py +71 -0
  14. raglab_bak/stores/repos.py +39 -0
  15. raglab_bak/stores/simple_stores.py +64 -0
  16. raglab_bak/stores/sql_app_stores.py +67 -0
  17. raglab_bak/stores/stores_util.py +128 -0
  18. raglab_bak/stores/user_store.py +9 -0
  19. raglab_bak/suggestion_engine/__init__.py +0 -0
  20. raglab_bak/suggestion_engine/suggestion_example.ipynb +317 -0
  21. raglab_bak/suggestion_engine/suggestion_lib.py +156 -0
  22. raglab_bak/templates/chat.html +131 -0
  23. raglab_bak/templates/gate.html +31 -0
  24. raglab_bak/templates/login.html +37 -0
  25. raglab_bak/templates/raglab_home.html +33 -0
  26. raglab_bak/templates/raglab_query.html +69 -0
  27. raglab_bak/templates/raglab_query_result.html +42 -0
  28. raglab_bak/tests/test.db +0 -0
  29. raglab_bak/tests/test_app.py +65 -0
  30. raglab_bak/tests/test_repo.py +0 -0
  31. raglab_bak/tests/test_stores_util.py +60 -0
  32. raglab_bak/transformations.py +25 -0
  33. raglab_bak/util.py +217 -0
  34. raglab_bak/web_services/__init__.py +1 -0
  35. raglab_bak/web_services/mockups.py +143 -0
  36. raglab_bak/web_services/prompt_pilote.py +73 -0
  37. raglab_bak/web_services/prompt_templates_dde.py +110 -0
  38. raglab_bak/web_services/store_access.py +47 -0
  39. raglab_bak/wips/__init__.py +1 -0
  40. raglab_bak/wips/raglab_wips.ipynb +299 -0
  41. raglab_bak/wips/vd_facade_1/__init__.py +10 -0
  42. raglab_bak/wips/vd_facade_1/components.py +88 -0
  43. raglab_bak/wips/vd_facade_1/config.py +31 -0
  44. raglab_bak/wips/vd_facade_1/dev_instructions_1.md +843 -0
  45. raglab_bak/wips/vd_facade_1/dev_notes.md +6 -0
  46. raglab_bak/wips/vd_facade_1/interface.py +78 -0
  47. raglab_bak/wips/vd_facade_1/langchain_integration.py +329 -0
  48. raglab_bak/wips/vd_facade_1/registry.py +71 -0
  49. raglab_bak/wips/vd_facade_1/tests/__init__.py +1 -0
  50. raglab_bak/wips/vd_facade_1/tests/test_basic.py +23 -0
  51. raglab_bak/wips/vd_facade_1/tests/test_integration.py +207 -0
  52. raglab_bak/wips/vd_facade_1/tests/test_qdrant_integration.py +62 -0
  53. raglab_bak/wips/vd_facade_1/types.py +73 -0
  54. raglab_bak/wips/vectordb_discovery.py +490 -0
  55. raglab_bak/wips/vectordb_manager.py +1162 -0
  56. raglab_bak-0.1.3.dist-info/METADATA +71 -0
  57. raglab_bak-0.1.3.dist-info/RECORD +59 -0
  58. raglab_bak-0.1.3.dist-info/WHEEL +4 -0
  59. raglab_bak-0.1.3.dist-info/licenses/LICENSE +21 -0
raglab_bak/Dockerfile ADDED
@@ -0,0 +1,12 @@
1
+ FROM python:3.9
2
+
3
+ WORKDIR /app
4
+
5
+ COPY . /app
6
+
7
+ COPY .env /app
8
+ COPY test.db /app
9
+
10
+ RUN pip install fastapi gpt4all openai langchain chromadb lkj config2py dill dol tiktoken oa ju psycopg2
11
+ RUN pip install git+https://github.com/i2mint/sqldol
12
+ CMD ["python", "app.py"]
raglab_bak/__init__.py ADDED
@@ -0,0 +1,3 @@
1
+ """A medley of tools to make RAG-based applications."""
2
+
3
+ from raglab_bak.util import LazyAccessor
raglab_bak/audio.py ADDED
@@ -0,0 +1,18 @@
1
+ """Audio utilities for RAG ingestion (e.g. silence removal)."""
2
+
3
+ from pydub import AudioSegment, silence
4
+
5
+
6
+ def remove_silence_from_audio(audio_file_path, silence_threshold=-50.0, chunk_size=10):
7
+ audio = AudioSegment.from_file(audio_file_path)
8
+
9
+ non_silent_parts = silence.detect_nonsilent(
10
+ audio, min_silence_len=chunk_size, silence_thresh=silence_threshold
11
+ )
12
+
13
+ processed_audio = AudioSegment.silent(duration=0)
14
+
15
+ for start, end in non_silent_parts:
16
+ processed_audio += audio[start:end]
17
+
18
+ return processed_audio
@@ -0,0 +1,32 @@
1
+ """Fiddle manager service: CRUD over per-user named objects, served over HTTP via py2http."""
2
+
3
+ from stores.repos import RepoJson
4
+ from py2http import mk_flat, run_app
5
+
6
+
7
+ class FiddleManager:
8
+ def __init__(self):
9
+ self.repo = RepoJson()
10
+
11
+ def create(self, user_id: str, name: str):
12
+ print("Creating object")
13
+ self.repo.create(user_id + name, "")
14
+
15
+ def get(self, user_id: str, name: str) -> str:
16
+ return self.repo.get(user_id + name)
17
+
18
+ def update(self, user_id: str, name: str, value: str):
19
+ return self.repo.update(user_id + name, value)
20
+
21
+ def delete(self, user_id: str, name: str):
22
+ return self.repo.delete(user_id + name)
23
+
24
+
25
+ def run_fiddle_manager():
26
+ create = mk_flat(FiddleManager, FiddleManager.create, func_name="create")
27
+ get = mk_flat(FiddleManager, FiddleManager.get, func_name="get")
28
+ update = mk_flat(FiddleManager, FiddleManager.update, func_name="update")
29
+ delete = mk_flat(FiddleManager, FiddleManager.delete, func_name="delete")
30
+ fns = [create, get, update, delete]
31
+
32
+ run_app(fns, publish_openapi=True, publish_swagger=True)
@@ -0,0 +1,13 @@
1
+ ```mermaid
2
+ sequenceDiagram
3
+ participant Frontend
4
+ participant Backend
5
+ Frontend->>Backend: GET /prompt
6
+ Backend-->>Frontend: JSON { [{name:str}] } (liste des templates)
7
+ Frontend->>Backend: GET /prompt/editor?name (récupérer le template + les forms)
8
+ Backend-->>Frontend: JSON {template:str, rjsf_ui:str} (template + ui)
9
+ Frontend->>Backend: POST /prompt/editor JSON (poster le template pour le sauvegarder)
10
+ Backend-->>Frontend: JSON {template, rjsf_ui} (template + rjsf_ui mis à jour)
11
+ Frontend->>Backend: GET /prompt/editor/ask?query
12
+ Backend-->>Frontend: JSON { answer:str }
13
+ ```
raglab_bak/raglab2.py ADDED
@@ -0,0 +1,174 @@
1
+ """
2
+ Taking the use case implemented by example:
3
+ https://python.langchain.com/docs/expression_language/cookbook/retrieval
4
+ I created a class that encapsulates the logic of the example, and added some
5
+ features to it. The class is called Raglab2.
6
+
7
+
8
+ Example of use:
9
+
10
+ ```python
11
+ from srag.tw.raglab2 import *
12
+ raglab = raglab2('test')
13
+
14
+ text_store = {
15
+ "doc1": "The Eiffel Tower is a wrought-iron lattice tower on the Champ de Mars in Paris, France. It is named after the engineer Gustave Eiffel, whose company designed and built the tower.",
16
+ "doc2": "The Great Wall of China is a series of fortifications made of stone, brick, tamped earth, wood, and other materials, generally built along an east-to-west line across the historical northern borders of China.",
17
+ "doc3": "The light bulb was invented by Thomas Edison in the 19th century. It was a groundbreaking invention that allowed for the widespread use of electric light in homes and businesses.",
18
+ "doc4": "Photosynthesis is a process used by plants, algae, and certain bacteria to convert light energy into chemical energy stored in glucose, which can be used to fuel the organism's activities."
19
+ }
20
+
21
+ # text_store = wrap_in_langchain_document(text_store)
22
+
23
+ raglab.update_with_mapping(text_store)
24
+
25
+ raglab.ask("Who invented the light bulb and what is the process plants use to convert light energy?")
26
+ ```
27
+
28
+ """
29
+
30
+ from typing import Optional, Union
31
+ from collections.abc import Callable, Mapping
32
+ from functools import cached_property, partial
33
+ import os
34
+ import chromadb
35
+ from chromadb.config import Settings
36
+
37
+ from dotenv import load_dotenv
38
+
39
+ from lkj import import_object, chunker, clog # pip install lkj
40
+ from config2py import get_configs_folder_for_app # pip install config2py
41
+
42
+ from langchain_core.embeddings import Embeddings
43
+ from langchain_core.language_models.llms import LLM
44
+ from langchain_core.output_parsers import StrOutputParser
45
+ from langchain_core.prompts import ChatPromptTemplate
46
+ from langchain_core.runnables import RunnableLambda, RunnablePassthrough
47
+ from langchain_core.vectorstores import VectorStore
48
+
49
+ from langchain_community.llms.gpt4all import GPT4All
50
+ from langchain_community.embeddings.gpt4all import GPT4AllEmbeddings
51
+ from langchain_community.vectorstores.chroma import Chroma
52
+
53
+ from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
54
+ from langchain.chains import LLMChain
55
+ from langchain.prompts import PromptTemplate
56
+
57
+ from stores.repos import RepoJson
58
+ from util import StoreAccess
59
+
60
+ segmenter = partial(chunker, chk_size=1000, include_tail=True)
61
+
62
+ DFLT_VECTORSTORES_PERSIST_ROOTDIR = get_configs_folder_for_app(
63
+ "srag", configs_name="vectorstores"
64
+ )
65
+
66
+ DFLT_PROMPT_TEMPLATE = """Answer the question based only on the following context:
67
+ {context}
68
+
69
+ Question: {question}
70
+ """
71
+
72
+ DFLT_PROMPT = ChatPromptTemplate.from_template(DFLT_PROMPT_TEMPLATE)
73
+
74
+
75
+ class Raglab2:
76
+ def __init__(
77
+ self,
78
+ vectorstore: VectorStore,
79
+ embedding: Embeddings,
80
+ prompt: PromptTemplate,
81
+ model: LLM, # orca-mini-3b-gguf2-q4_0 all-MiniLM-L6-v2-f16.gguf orca-mini-3b-gguf2-q4_0.gguf mistral-7b-openorca.Q4_0.gguf
82
+ ):
83
+ self.vectorstore = vectorstore
84
+ self.embedding = embedding
85
+ self.prompt = prompt
86
+ self.model = model
87
+
88
+ def update_with_mapping(
89
+ self,
90
+ mapping: Mapping[str, str],
91
+ *,
92
+ chk_size: int = 1000,
93
+ verbose: bool = False,
94
+ # filt: Callable = None # to control update rules
95
+ ):
96
+
97
+ ids, texts = map(list, zip(*mapping.items()))
98
+ self.vectorstore.from_texts(texts=texts, ids=ids, embedding=self.embedding)
99
+
100
+ # TODO: Make the segmented (batched) update of the vectorstore work
101
+ # --> Need to find the right vectorstore method for that (maybe update_documents?)
102
+ # TODO: Figure out how to only update vectorstores when "docs" have changed
103
+ # (editted, deleted, added, etc.)
104
+ # _segmenter = partial(segmenter, chk_size=chk_size)
105
+ # _clog = clog(verbose, log_func=lambda x: print(f"segment {i}"))
106
+
107
+ # for i, _items in enumerate(_segmenter(mapping.items())):
108
+ # _clog(i)
109
+ # ids, texts = map(list, zip(*_items))
110
+ # self.vectorstore.from_texts(texts=texts, ids=ids, embedding=self.embedding)
111
+
112
+ return self
113
+
114
+ @cached_property
115
+ def retriever(self):
116
+ return self.vectorstore.as_retriever()
117
+
118
+ @cached_property
119
+ def chain(self):
120
+ return (
121
+ {"context": self.retriever, "question": RunnablePassthrough()}
122
+ | self.prompt
123
+ | self.model
124
+ | StrOutputParser()
125
+ )
126
+
127
+ def ask(self, query: str):
128
+ return self.chain.invoke(query)
129
+
130
+
131
+ class RaglabSession:
132
+ def __init__(self, app: Raglab2):
133
+ self.raglab_app = app
134
+
135
+ def ask(self, query: str):
136
+ return self.raglab_app.ask(quer)
137
+
138
+
139
+ class RaglabSessionBuilder:
140
+ def build(
141
+ user: str,
142
+ name: str,
143
+ vectorstore: VectorStore,
144
+ embedding: Embeddings,
145
+ prompt: PromptTemplate,
146
+ model: LLM, # orca-mini-3b-gguf2-q4_0 all-MiniLM-L6-v2-f16.gguf orca-mini-3b-gguf2-q4_0.gguf mistral-7b-openorca.Q4_0.gguf
147
+ ) -> RaglabSession:
148
+
149
+ raglab = Raglab2(vectorstore, embedding, prompt, model)
150
+ session = RaglabSession(raglab)
151
+ session.raglab_app = raglab
152
+
153
+ return session
154
+
155
+ def restore(user: str, name: str) -> RaglabSession:
156
+ pass
157
+
158
+
159
+ from langchain.docstore.document import Document
160
+ from dol import wrap_kvs
161
+
162
+
163
+ def _wrap_kv_in_langchain_document(k, v):
164
+ return Document(page_content=v, metadata={"key": k})
165
+
166
+
167
+ def wrap_in_langchain_document(mapping: Mapping[str, str]):
168
+ return wrap_kvs(mapping, postget=_wrap_kv_in_langchain_document)
169
+
170
+
171
+ def object_resolver(module_dot_path, obj_name=None, *args, **kwargs):
172
+ if obj_name is None:
173
+ return partial(object_resolver, module_dot_path)
174
+ return import_object(".".join([module_dot_path, obj_name]))
@@ -0,0 +1,163 @@
1
+ """VectorDB"""
2
+
3
+ # VectorDB class
4
+ from typing import Dict, Any
5
+ from collections.abc import Mapping, Iterable
6
+ from dataclasses import dataclass, field
7
+ from types import SimpleNamespace
8
+ from langchain.text_splitter import RecursiveCharacterTextSplitter
9
+ from functools import cached_property
10
+
11
+ DocKey = str
12
+ DocValue = str
13
+
14
+
15
+ from typing import Tuple, List
16
+ from collections.abc import Mapping
17
+
18
+ SegmentKey = tuple[DocKey, int, int]
19
+
20
+
21
+ def generate_split_keys(
22
+ docs: dict, text_splitter: RecursiveCharacterTextSplitter, metadatas: list[dict]
23
+ ) -> list[tuple[str, int, int]]:
24
+
25
+ documents = text_splitter.create_documents(
26
+ list(docs.values()), metadatas=list({"document_name": k} for k in docs.keys())
27
+ )
28
+
29
+ return [
30
+ (
31
+ d.metadata["document_name"],
32
+ d.metadata["start_index"],
33
+ d.metadata["start_index"] + len(d.page_content),
34
+ )
35
+ for d in documents
36
+ ]
37
+
38
+
39
+ class SegmentMapping:
40
+ """A class to represent a mapping between segments and documents."""
41
+
42
+ def __init__(self, docs: Mapping, segment_keys: list[SegmentKey]):
43
+ self.docs = docs
44
+ self.segment_keys = segment_keys
45
+ self.document_keys = list(docs.keys())
46
+
47
+ def __iter__(self):
48
+ yield from self.segment_keys
49
+
50
+ def __getitem__(self, key: SegmentKey):
51
+ if isinstance(key, str):
52
+ return self.docs[key]
53
+ elif isinstance(key, tuple):
54
+ doc_key, start_idx, end_idx = key
55
+ return self.docs[doc_key][start_idx:end_idx]
56
+ else:
57
+ raise TypeError("Key must be a string or a tuple")
58
+
59
+ def __setitem__(self, key: SegmentKey, value: str):
60
+ if isinstance(key, str):
61
+ self.docs[key] = value
62
+ return
63
+ else:
64
+ doc_key, start_idx, end_idx = key
65
+ self.segment_keys.append(key)
66
+ self.docs[doc_key] = (
67
+ self.docs.get(doc_key, "")[:start_idx]
68
+ + value
69
+ + self.docs.get(doc_key, "")[end_idx:]
70
+ )
71
+
72
+ def __add__(self, other):
73
+ """Add two SegmentMapping objects together. This will concatenate the documents and segment keys."""
74
+ return SegmentMapping(
75
+ {**self.docs, **other.docs}, self.segment_keys + other.segment_keys
76
+ )
77
+
78
+ def __len__(self):
79
+ return len(self.segment_keys)
80
+
81
+ def __contains__(self, key: SegmentKey):
82
+ if isinstance(key, str):
83
+ return key in self.document_keys
84
+ elif isinstance(key, tuple):
85
+ return key in self.segment_keys
86
+ else:
87
+ raise TypeError("Key must be a string or a tuple")
88
+
89
+ def __repr__(self):
90
+ representation = ""
91
+ for key in self.segment_keys:
92
+ representation += str(key) + " : " + str(self.__getitem__(key)) + "\n"
93
+ return representation
94
+
95
+ def values(self):
96
+ for key in self.segment_keys:
97
+ yield self.__getitem__(key)
98
+
99
+
100
+ @dataclass
101
+ class ChunkDB:
102
+ """Contains a mapping from document keys to document content. Keys can either be a tuple (doc_key, start_idx, end_idx) or a string doc_key.
103
+ The documents are split into segments and the segments are stored in a SegmentMapping object.
104
+ The SegmentMapping object is a mapping from segment keys to segments."""
105
+
106
+ docs: Mapping[DocKey, DocValue]
107
+ # kwargs: Dict[str, Any] = field(default_factory=dict)
108
+ chunk_size: int = 400
109
+ chunk_overlap: int = 100
110
+
111
+ @cached_property
112
+ def segments(self):
113
+ return self.mk_segment_store(self.docs)
114
+
115
+ @cached_property
116
+ def segment_store(self):
117
+ return SimpleNamespace(segment_keys=self.segments)
118
+
119
+ @cached_property
120
+ def document_store(self):
121
+ return SimpleNamespace(docs=self.docs)
122
+
123
+ @cached_property
124
+ def mall(self):
125
+ return SimpleNamespace(docs=self.docs, segment_keys=self.segments)
126
+
127
+ def add(self, new_documents: Mapping[DocKey, DocValue], when_exists: str = "raise"):
128
+ """Add new documents to the database. When_exists can be "raise", "skip" or "overwrite.
129
+ New documents has to be a mapping with the document key as the key and the document content as the value.
130
+ """
131
+
132
+ for new_doc_key in new_documents:
133
+ if new_doc_key in self.segments:
134
+ if when_exists == "skip":
135
+ continue
136
+ elif when_exists == "overwrite":
137
+ self.segments[new_doc_key] = new_documents[new_doc_key]
138
+ else:
139
+ raise ValueError(
140
+ f"Document with key {new_doc_key} already exists. Use 'when_exists' to handle this case."
141
+ )
142
+ else:
143
+ self.segments = self.segments + self.mk_segment_store(new_documents)
144
+
145
+ def mk_segment_store(self, docs):
146
+ """Creates and returns a segment store from a dictionary of documents. Segment store is a mapping from segment keys to segments."""
147
+ text_splitter = RecursiveCharacterTextSplitter(
148
+ chunk_size=self.chunk_size,
149
+ chunk_overlap=min(self.chunk_overlap, self.chunk_size - 1),
150
+ length_function=len,
151
+ add_start_index=True, # enforce the start index
152
+ is_separator_regex=False,
153
+ )
154
+ segment_keys = generate_split_keys(docs, text_splitter, metadatas=[])
155
+ return SegmentMapping(docs, segment_keys)
156
+
157
+ def search(self, query: str, k: int = 1) -> Iterable[DocKey]:
158
+ # TODO : return (yields) the k most similar documents to the query
159
+ pass
160
+
161
+ def add_metadata(self, metadata: Mapping[DocKey, dict]):
162
+ # TODO : add metadata to the documents
163
+ pass
File without changes
@@ -0,0 +1,31 @@
1
+ """Client wrapper for connecting to a remote Chroma vector database."""
2
+
3
+ import os
4
+ import chromadb
5
+ from chromadb.config import Settings
6
+ from dotenv import load_dotenv
7
+
8
+
9
+ class ChromaClient:
10
+ def __init__(self):
11
+ load_dotenv()
12
+ chroma_key = os.getenv("CHROMA_KEY")
13
+ self.client = chromadb.HttpClient(
14
+ host="149.202.47.109",
15
+ port="45000",
16
+ settings=Settings(
17
+ chroma_client_auth_provider="chromadb.auth.token.TokenAuthClientProvider",
18
+ chroma_client_auth_credentials=chroma_key,
19
+ ),
20
+ )
21
+
22
+ def get_collections(self):
23
+ collections = self.client.list_collections()
24
+ return collections
25
+
26
+ def add_collection(self, name: str):
27
+ self.client.create_collection(name)
28
+
29
+ def query(self, dataset_name: str, query: str):
30
+ collection = self.client.get_collection(dataset_name)
31
+ return collection.query(query_texts=query)