sunholo 0.67.8__py3-none-any.whl → 0.68.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sunholo/agents/flask/qna_routes.py +9 -3
- sunholo/components/retriever.py +11 -6
- sunholo/database/alloydb.py +3 -183
- sunholo/database/alloydb_client.py +196 -0
- sunholo/gcs/add_file.py +4 -0
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/METADATA +2 -2
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/RECORD +11 -10
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/LICENSE.txt +0 -0
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/WHEEL +0 -0
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/entry_points.txt +0 -0
- {sunholo-0.67.8.dist-info → sunholo-0.68.0.dist-info}/top_level.txt +0 -0
|
@@ -258,7 +258,12 @@ def register_qna_routes(app, stream_interpreter, vac_interpreter):
|
|
|
258
258
|
# the header forwarded
|
|
259
259
|
auth_header = request.headers.get('X-Forwarded-Authorization')
|
|
260
260
|
if auth_header:
|
|
261
|
-
|
|
261
|
+
|
|
262
|
+
if auth_header.startswith('Bearer '):
|
|
263
|
+
api_key = auth_header.split(' ')[1] # Assuming "Bearer <api_key>"
|
|
264
|
+
else:
|
|
265
|
+
return jsonify({'error': 'Invalid authorization header does not start with "Bearer " - got: {auth_header}'}), 401
|
|
266
|
+
|
|
262
267
|
endpoints_host = os.getenv('_ENDPOINTS_HOST')
|
|
263
268
|
if not endpoints_host:
|
|
264
269
|
return jsonify({'error': '_ENDPOINTS_HOST environment variable not found'}), 401
|
|
@@ -425,8 +430,9 @@ def register_qna_routes(app, stream_interpreter, vac_interpreter):
|
|
|
425
430
|
return make_openai_response(user_message, vector_name, 'ERROR: could not find an answer')
|
|
426
431
|
|
|
427
432
|
except Exception as err:
|
|
428
|
-
log.error(f"OpenAI response error: {err}")
|
|
429
|
-
|
|
433
|
+
log.error(f"OpenAI response error: {str(err)} traceback: {traceback.format_exc()}")
|
|
434
|
+
|
|
435
|
+
return make_openai_response(user_message, vector_name, f'ERROR: {str(err)}')
|
|
430
436
|
|
|
431
437
|
|
|
432
438
|
def create_langfuse_trace(request, vector_name):
|
sunholo/components/retriever.py
CHANGED
|
@@ -53,11 +53,16 @@ def pick_retriever(vector_name, embeddings=None):
|
|
|
53
53
|
continue
|
|
54
54
|
|
|
55
55
|
embeddings = embeddings or get_embeddings(vector_name)
|
|
56
|
-
read_only = value.get('
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
56
|
+
read_only = value.get('read_only')
|
|
57
|
+
try:
|
|
58
|
+
vectorstore = pick_vectorstore(vectorstore,
|
|
59
|
+
vector_name=vector_name,
|
|
60
|
+
embeddings=embeddings,
|
|
61
|
+
read_only=read_only)
|
|
62
|
+
except Exception as e:
|
|
63
|
+
log.error(f"Failed to pick_vectorstore {vectorstore} for {vector_name} - {str(e)} - skipping")
|
|
64
|
+
continue
|
|
65
|
+
|
|
61
66
|
k_override = value.get('k', 3)
|
|
62
67
|
vs_retriever = vectorstore.as_retriever(search_kwargs=dict(k=k_override))
|
|
63
68
|
retriever_list.append(vs_retriever)
|
|
@@ -103,7 +108,7 @@ def metadata_retriever(metadata: dict, key: str, vector_name:str, embeddings=Non
|
|
|
103
108
|
if key not in metadata:
|
|
104
109
|
raise ValueError(f"Missing {key} in {metadata}")
|
|
105
110
|
the_id = metadata[key]
|
|
106
|
-
read_only = value.get('
|
|
111
|
+
read_only = value.get('read_only')
|
|
107
112
|
embeddings = embeddings or get_embeddings(vector_name)
|
|
108
113
|
vectorstore = pick_vectorstore(vectorstore,
|
|
109
114
|
vector_name=the_id,
|
sunholo/database/alloydb.py
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
import os
|
|
2
2
|
try:
|
|
3
|
-
import pg8000
|
|
4
|
-
import sqlalchemy
|
|
5
3
|
from sqlalchemy.exc import DatabaseError, ProgrammingError
|
|
6
4
|
from asyncpg.exceptions import DuplicateTableError
|
|
7
|
-
from google.cloud.alloydb.connector import Connector
|
|
8
5
|
from langchain_google_alloydb_pg import AlloyDBEngine, Column, AlloyDBLoader, AlloyDBDocumentSaver
|
|
9
6
|
from google.cloud.alloydb.connector import IPTypes
|
|
10
7
|
except ImportError:
|
|
@@ -12,9 +9,12 @@ except ImportError:
|
|
|
12
9
|
pass
|
|
13
10
|
|
|
14
11
|
from .database import get_vector_size
|
|
12
|
+
from .alloydb_client import AlloyDBClient
|
|
13
|
+
|
|
15
14
|
from ..logging import log
|
|
16
15
|
from ..utils.config import load_config_key
|
|
17
16
|
|
|
17
|
+
|
|
18
18
|
def create_alloydb_engine(vector_name):
|
|
19
19
|
|
|
20
20
|
if not AlloyDBEngine:
|
|
@@ -49,183 +49,6 @@ def create_alloydb_engine(vector_name):
|
|
|
49
49
|
|
|
50
50
|
return engine
|
|
51
51
|
|
|
52
|
-
class AlloyDBClient:
|
|
53
|
-
"""
|
|
54
|
-
A class to manage interactions with an AlloyDB instance.
|
|
55
|
-
|
|
56
|
-
Example Usage:
|
|
57
|
-
|
|
58
|
-
```python
|
|
59
|
-
client = AlloyDBClient(
|
|
60
|
-
project_id="your-project-id",
|
|
61
|
-
region="your-region",
|
|
62
|
-
cluster_name="your-cluster-name",
|
|
63
|
-
instance_name="your-instance-name",
|
|
64
|
-
user="your-db-user",
|
|
65
|
-
password="your-db-password"
|
|
66
|
-
)
|
|
67
|
-
|
|
68
|
-
# Create a database
|
|
69
|
-
client.execute_sql("CREATE DATABASE my_database")
|
|
70
|
-
|
|
71
|
-
# Execute other SQL statements
|
|
72
|
-
client.execute_sql("CREATE TABLE my_table (id INT, name VARCHAR(50))")
|
|
73
|
-
```
|
|
74
|
-
"""
|
|
75
|
-
|
|
76
|
-
def __init__(self,
|
|
77
|
-
project_id: str,
|
|
78
|
-
region: str,
|
|
79
|
-
cluster_name:str,
|
|
80
|
-
instance_name:str,
|
|
81
|
-
user:str,
|
|
82
|
-
password=None,
|
|
83
|
-
db="postgres"):
|
|
84
|
-
"""Initializes the AlloyDB client.
|
|
85
|
-
- project_id (str): GCP project ID where the AlloyDB instance resides.
|
|
86
|
-
- region (str): The region where the AlloyDB instance is located.
|
|
87
|
-
- cluster_name (str): The name of the AlloyDB cluster.
|
|
88
|
-
- instance_name (str): The name of the AlloyDB instance.
|
|
89
|
-
- user (str): The database user name.
|
|
90
|
-
- password (str): The database user's password.
|
|
91
|
-
- db_name (str): The name of the database.
|
|
92
|
-
"""
|
|
93
|
-
self.connector = Connector()
|
|
94
|
-
self.inst_uri = self._build_instance_uri(project_id, region, cluster_name, instance_name)
|
|
95
|
-
self.engine = self._create_engine(self.inst_uri, user, password, db)
|
|
96
|
-
|
|
97
|
-
def _build_instance_uri(self, project_id, region, cluster_name, instance_name):
|
|
98
|
-
return f"projects/{project_id}/locations/{region}/clusters/{cluster_name}/instances/{instance_name}"
|
|
99
|
-
|
|
100
|
-
def _create_engine(self, inst_uri, user, password, db):
|
|
101
|
-
def getconn() -> pg8000.dbapi.Connection:
|
|
102
|
-
conn = self.connector.connect(
|
|
103
|
-
inst_uri,
|
|
104
|
-
"pg8000",
|
|
105
|
-
user=user,
|
|
106
|
-
password=password,
|
|
107
|
-
db=db,
|
|
108
|
-
enable_iam_auth=True,
|
|
109
|
-
)
|
|
110
|
-
return conn
|
|
111
|
-
|
|
112
|
-
engine = sqlalchemy.create_engine(
|
|
113
|
-
"postgresql+pg8000://",
|
|
114
|
-
isolation_level="AUTOCOMMIT",
|
|
115
|
-
creator=getconn
|
|
116
|
-
)
|
|
117
|
-
engine.dialect.description_encoding = None
|
|
118
|
-
log.info(f"Created AlloyDB engine for {inst_uri} and user: {user}")
|
|
119
|
-
return engine
|
|
120
|
-
|
|
121
|
-
def execute_sql(self, sql_statement):
|
|
122
|
-
"""Executes a given SQL statement with error handling.
|
|
123
|
-
|
|
124
|
-
- sql_statement (str): The SQL statement to execute.
|
|
125
|
-
- Returns: The result of the execution, if any.
|
|
126
|
-
"""
|
|
127
|
-
sql_ = sqlalchemy.text(sql_statement)
|
|
128
|
-
result = None
|
|
129
|
-
with self.engine.connect() as conn:
|
|
130
|
-
try:
|
|
131
|
-
log.info(f"Executing SQL statement: {sql_}")
|
|
132
|
-
result = conn.execute(sql_)
|
|
133
|
-
except DatabaseError as e:
|
|
134
|
-
if "already exists" in str(e):
|
|
135
|
-
log.warning(f"Error ignored: {str(e)}. Assuming object already exists.")
|
|
136
|
-
else:
|
|
137
|
-
raise
|
|
138
|
-
finally:
|
|
139
|
-
conn.close()
|
|
140
|
-
|
|
141
|
-
return result
|
|
142
|
-
|
|
143
|
-
@staticmethod
|
|
144
|
-
def _and_or_ilike(sources, search_type="OR", operator="ILIKE"):
|
|
145
|
-
unique_sources = set(sources)
|
|
146
|
-
# Choose the delimiter based on the search_type argument
|
|
147
|
-
delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
|
|
148
|
-
|
|
149
|
-
# Build the conditional expressions based on the chosen delimiter
|
|
150
|
-
conditions = delimiter.join(f"TRIM(source) {operator} '%{source}%'" for source in unique_sources)
|
|
151
|
-
if not conditions:
|
|
152
|
-
log.warning("Alloydb doc query found no like_patterns")
|
|
153
|
-
return []
|
|
154
|
-
|
|
155
|
-
return conditions
|
|
156
|
-
|
|
157
|
-
def delete_sources_from_alloydb(self, sources, vector_name):
|
|
158
|
-
"""
|
|
159
|
-
Deletes from both vectorstore and docstore
|
|
160
|
-
"""
|
|
161
|
-
|
|
162
|
-
vector_length = get_vector_size(vector_name)
|
|
163
|
-
|
|
164
|
-
conditions = self._and_or_ilike(sources, operator="=")
|
|
165
|
-
|
|
166
|
-
if not conditions:
|
|
167
|
-
log.warning("No conditions were specified, not deleting whole table!")
|
|
168
|
-
return False
|
|
169
|
-
|
|
170
|
-
query = f"""
|
|
171
|
-
DELETE FROM {vector_name}_docstore
|
|
172
|
-
WHERE {conditions};
|
|
173
|
-
DELETE FROM {vector_name}_vectorstore_{vector_length}
|
|
174
|
-
WHERE {conditions}
|
|
175
|
-
"""
|
|
176
|
-
|
|
177
|
-
return self.execute_sql(query)
|
|
178
|
-
|
|
179
|
-
def create_database(self, database_name):
|
|
180
|
-
self.execute_sql(f'CREATE DATABASE "{database_name}"')
|
|
181
|
-
|
|
182
|
-
def fetch_owners(self):
|
|
183
|
-
owners = self.execute_sql('SELECT table_schema, table_name, privilege_type FROM information_schema.table_privileges')
|
|
184
|
-
for row in owners:
|
|
185
|
-
print(f"Schema: {row[0]}, Table: {row[1]}, Privilege: {row[2]}")
|
|
186
|
-
return owners
|
|
187
|
-
|
|
188
|
-
def create_schema(self, schema_name="public"):
|
|
189
|
-
self.execute_sql(f'CREATE SCHEMA IF NOT EXISTS {schema_name};')
|
|
190
|
-
|
|
191
|
-
def grant_permissions(self, schema_name, users):
|
|
192
|
-
for user in users:
|
|
193
|
-
self.execute_sql(f'GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema_name} TO "{user}";')
|
|
194
|
-
self.execute_sql(f'GRANT USAGE, CREATE ON SCHEMA {schema_name} TO "{user}";')
|
|
195
|
-
self.execute_sql(f'ALTER DEFAULT PRIVILEGES IN SCHEMA {schema_name} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO "{user}";')
|
|
196
|
-
self.execute_sql(f'GRANT USAGE ON SCHEMA information_schema TO "{user}";')
|
|
197
|
-
self.execute_sql(f'GRANT SELECT ON information_schema.columns TO "{user}";')
|
|
198
|
-
|
|
199
|
-
def create_docstore_tables(self, vector_names, users):
|
|
200
|
-
for vector_name in vector_names:
|
|
201
|
-
table_name = f"{vector_name}_docstore"
|
|
202
|
-
sql = f'''
|
|
203
|
-
CREATE TABLE IF NOT EXISTS "{table_name}"
|
|
204
|
-
(page_content TEXT, doc_id UUID, source TEXT, images_gsurls JSONB, chunk_metadata JSONB, langchain_metadata JSONB)
|
|
205
|
-
'''
|
|
206
|
-
self.execute_sql(sql)
|
|
207
|
-
|
|
208
|
-
for user in users:
|
|
209
|
-
self.execute_sql(f'GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "{table_name}" TO "{user}";')
|
|
210
|
-
|
|
211
|
-
vectorstore_id = f"{vector_name}_vectorstore_1536"
|
|
212
|
-
sql = f'''
|
|
213
|
-
CREATE TABLE IF NOT EXISTS "{vectorstore_id}" (
|
|
214
|
-
langchain_id UUID NOT NULL,
|
|
215
|
-
content TEXT NOT NULL,
|
|
216
|
-
embedding vector NOT NULL,
|
|
217
|
-
source TEXT,
|
|
218
|
-
langchain_metadata JSONB,
|
|
219
|
-
docstore_doc_id UUID,
|
|
220
|
-
eventTime TIMESTAMPTZ
|
|
221
|
-
);
|
|
222
|
-
'''
|
|
223
|
-
self.execute_sql(sql)
|
|
224
|
-
|
|
225
|
-
for user in users:
|
|
226
|
-
self.execute_sql(f'GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE {vectorstore_id} TO "{user}";')
|
|
227
|
-
|
|
228
|
-
|
|
229
52
|
alloydb_table_cache = {} # Our cache, initially empty # noqa: F841
|
|
230
53
|
def create_alloydb_table(vector_name, engine, type = "vectorstore", alloydb_config=None, username=None):
|
|
231
54
|
global alloydb_table_cache
|
|
@@ -240,9 +63,6 @@ def create_alloydb_table(vector_name, engine, type = "vectorstore", alloydb_conf
|
|
|
240
63
|
|
|
241
64
|
return table_name
|
|
242
65
|
|
|
243
|
-
alloydb_table_cache[table_name] = True
|
|
244
|
-
return table_name
|
|
245
|
-
|
|
246
66
|
log.info(f"# Creating AlloyDB table {table_name}")
|
|
247
67
|
try:
|
|
248
68
|
engine.init_vectorstore_table(
|
|
@@ -0,0 +1,196 @@
|
|
|
1
|
+
try:
|
|
2
|
+
import pg8000
|
|
3
|
+
import sqlalchemy
|
|
4
|
+
from sqlalchemy.exc import DatabaseError, ProgrammingError
|
|
5
|
+
from google.cloud.alloydb.connector import Connector
|
|
6
|
+
except ImportError:
|
|
7
|
+
AlloyDBEngine = None
|
|
8
|
+
pass
|
|
9
|
+
|
|
10
|
+
from .database import get_vector_size
|
|
11
|
+
from ..logging import log
|
|
12
|
+
|
|
13
|
+
class AlloyDBClient:
|
|
14
|
+
"""
|
|
15
|
+
A class to manage interactions with an AlloyDB instance.
|
|
16
|
+
|
|
17
|
+
Example Usage:
|
|
18
|
+
|
|
19
|
+
```python
|
|
20
|
+
client = AlloyDBClient(
|
|
21
|
+
project_id="your-project-id",
|
|
22
|
+
region="your-region",
|
|
23
|
+
cluster_name="your-cluster-name",
|
|
24
|
+
instance_name="your-instance-name",
|
|
25
|
+
user="your-db-user",
|
|
26
|
+
password="your-db-password"
|
|
27
|
+
)
|
|
28
|
+
|
|
29
|
+
# Create a database
|
|
30
|
+
client.execute_sql("CREATE DATABASE my_database")
|
|
31
|
+
|
|
32
|
+
# Execute other SQL statements
|
|
33
|
+
client.execute_sql("CREATE TABLE my_table (id INT, name VARCHAR(50))")
|
|
34
|
+
```
|
|
35
|
+
"""
|
|
36
|
+
|
|
37
|
+
def __init__(self,
|
|
38
|
+
project_id: str,
|
|
39
|
+
region: str,
|
|
40
|
+
cluster_name:str,
|
|
41
|
+
instance_name:str,
|
|
42
|
+
user:str,
|
|
43
|
+
password=None,
|
|
44
|
+
db="postgres"):
|
|
45
|
+
"""Initializes the AlloyDB client.
|
|
46
|
+
- project_id (str): GCP project ID where the AlloyDB instance resides.
|
|
47
|
+
- region (str): The region where the AlloyDB instance is located.
|
|
48
|
+
- cluster_name (str): The name of the AlloyDB cluster.
|
|
49
|
+
- instance_name (str): The name of the AlloyDB instance.
|
|
50
|
+
- user (str): The database user name.
|
|
51
|
+
- password (str): The database user's password.
|
|
52
|
+
- db_name (str): The name of the database.
|
|
53
|
+
"""
|
|
54
|
+
self.connector = Connector()
|
|
55
|
+
self.inst_uri = self._build_instance_uri(project_id, region, cluster_name, instance_name)
|
|
56
|
+
self.engine = self._create_engine(self.inst_uri, user, password, db)
|
|
57
|
+
|
|
58
|
+
def _build_instance_uri(self, project_id, region, cluster_name, instance_name):
|
|
59
|
+
return f"projects/{project_id}/locations/{region}/clusters/{cluster_name}/instances/{instance_name}"
|
|
60
|
+
|
|
61
|
+
def _create_engine(self, inst_uri, user, password, db):
|
|
62
|
+
def getconn() -> pg8000.dbapi.Connection:
|
|
63
|
+
conn = self.connector.connect(
|
|
64
|
+
inst_uri,
|
|
65
|
+
"pg8000",
|
|
66
|
+
user=user,
|
|
67
|
+
password=password,
|
|
68
|
+
db=db,
|
|
69
|
+
enable_iam_auth=True,
|
|
70
|
+
)
|
|
71
|
+
return conn
|
|
72
|
+
|
|
73
|
+
engine = sqlalchemy.create_engine(
|
|
74
|
+
"postgresql+pg8000://",
|
|
75
|
+
isolation_level="AUTOCOMMIT",
|
|
76
|
+
creator=getconn
|
|
77
|
+
)
|
|
78
|
+
engine.dialect.description_encoding = None
|
|
79
|
+
log.info(f"Created AlloyDB engine for {inst_uri} and user: {user}")
|
|
80
|
+
return engine
|
|
81
|
+
|
|
82
|
+
def execute_sql(self, sql_statement):
|
|
83
|
+
"""Executes a given SQL statement with error handling.
|
|
84
|
+
|
|
85
|
+
- sql_statement (str): The SQL statement to execute.
|
|
86
|
+
- Returns: The result of the execution, if any.
|
|
87
|
+
"""
|
|
88
|
+
sql_ = sqlalchemy.text(sql_statement)
|
|
89
|
+
result = None
|
|
90
|
+
with self.engine.connect() as conn:
|
|
91
|
+
try:
|
|
92
|
+
log.info(f"Executing SQL statement: {sql_}")
|
|
93
|
+
result = conn.execute(sql_)
|
|
94
|
+
except DatabaseError as e:
|
|
95
|
+
if "already exists" in str(e):
|
|
96
|
+
log.warning(f"Error ignored: {str(e)}. Assuming object already exists.")
|
|
97
|
+
else:
|
|
98
|
+
raise
|
|
99
|
+
finally:
|
|
100
|
+
conn.close()
|
|
101
|
+
|
|
102
|
+
return result
|
|
103
|
+
|
|
104
|
+
@staticmethod
|
|
105
|
+
def _and_or_ilike(sources, search_type="OR", operator="ILIKE"):
|
|
106
|
+
unique_sources = set(sources)
|
|
107
|
+
# Choose the delimiter based on the search_type argument
|
|
108
|
+
delimiter = ' AND ' if search_type.upper() == "AND" else ' OR '
|
|
109
|
+
|
|
110
|
+
# Build the conditional expressions based on the chosen delimiter
|
|
111
|
+
conditions = delimiter.join(f"TRIM(source) {operator} '%{source}%'" for source in unique_sources)
|
|
112
|
+
if not conditions:
|
|
113
|
+
log.warning("Alloydb doc query found no like_patterns")
|
|
114
|
+
return []
|
|
115
|
+
|
|
116
|
+
return conditions
|
|
117
|
+
|
|
118
|
+
def delete_sources_from_alloydb(self, sources, vector_name):
|
|
119
|
+
"""
|
|
120
|
+
Deletes from both vectorstore and docstore
|
|
121
|
+
"""
|
|
122
|
+
|
|
123
|
+
vector_length = get_vector_size(vector_name)
|
|
124
|
+
|
|
125
|
+
conditions = self._and_or_ilike(sources, operator="=")
|
|
126
|
+
|
|
127
|
+
if not conditions:
|
|
128
|
+
log.warning("No conditions were specified, not deleting whole table!")
|
|
129
|
+
return False
|
|
130
|
+
|
|
131
|
+
query = f"""
|
|
132
|
+
DELETE FROM {vector_name}_docstore
|
|
133
|
+
WHERE {conditions};
|
|
134
|
+
DELETE FROM {vector_name}_vectorstore_{vector_length}
|
|
135
|
+
WHERE {conditions}
|
|
136
|
+
"""
|
|
137
|
+
|
|
138
|
+
return self.execute_sql(query)
|
|
139
|
+
|
|
140
|
+
def create_database(self, database_name):
|
|
141
|
+
self.execute_sql(f'CREATE DATABASE "{database_name}"')
|
|
142
|
+
|
|
143
|
+
def fetch_owners(self):
|
|
144
|
+
owners = self.execute_sql('SELECT table_schema, table_name, privilege_type FROM information_schema.table_privileges')
|
|
145
|
+
for row in owners:
|
|
146
|
+
print(f"Schema: {row[0]}, Table: {row[1]}, Privilege: {row[2]}")
|
|
147
|
+
return owners
|
|
148
|
+
|
|
149
|
+
def create_schema(self, schema_name="public"):
|
|
150
|
+
self.execute_sql(f'CREATE SCHEMA IF NOT EXISTS {schema_name};')
|
|
151
|
+
|
|
152
|
+
def grant_schema_permissions(self, schema_name, users):
|
|
153
|
+
for user in users:
|
|
154
|
+
self.execute_sql(f'GRANT SELECT, INSERT, UPDATE, DELETE ON ALL TABLES IN SCHEMA {schema_name} TO "{user}";')
|
|
155
|
+
self.execute_sql(f'GRANT USAGE, CREATE ON SCHEMA {schema_name} TO "{user}";')
|
|
156
|
+
self.execute_sql(f'ALTER DEFAULT PRIVILEGES IN SCHEMA {schema_name} GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO "{user}";')
|
|
157
|
+
self.execute_sql(f'GRANT USAGE ON SCHEMA information_schema TO "{user}";')
|
|
158
|
+
self.execute_sql(f'GRANT SELECT ON information_schema.columns TO "{user}";')
|
|
159
|
+
|
|
160
|
+
def grant_table_permissions(self, table_name, users):
|
|
161
|
+
for user in users:
|
|
162
|
+
self.execute_sql(f'GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE "{table_name}" TO "{user}";')
|
|
163
|
+
|
|
164
|
+
def create_tables(self, vector_name, users):
|
|
165
|
+
self.create_docstore_table(vector_name, users)
|
|
166
|
+
self.create_vectorstore_table(vector_name, users)
|
|
167
|
+
|
|
168
|
+
def create_docstore_table(self, vector_name: str, users):
|
|
169
|
+
table_name = f"{vector_name}_vectorstore"
|
|
170
|
+
sql = f'''
|
|
171
|
+
CREATE TABLE IF NOT EXISTS "{table_name}"
|
|
172
|
+
(page_content TEXT, doc_id UUID, source TEXT, images_gsurls JSONB, chunk_metadata JSONB, langchain_metadata JSONB)
|
|
173
|
+
'''
|
|
174
|
+
self.execute_sql(sql)
|
|
175
|
+
|
|
176
|
+
self.grant_table_permissions(table_name, users)
|
|
177
|
+
|
|
178
|
+
def create_vectorstore_table(self, vector_name: str, users):
|
|
179
|
+
from .database import get_vector_size
|
|
180
|
+
vector_size = get_vector_size(vector_name)
|
|
181
|
+
vectorstore_id = f"{vector_name}_{type}_{vector_size}"
|
|
182
|
+
|
|
183
|
+
sql = f'''
|
|
184
|
+
CREATE TABLE IF NOT EXISTS "{vectorstore_id}" (
|
|
185
|
+
langchain_id UUID NOT NULL,
|
|
186
|
+
content TEXT NOT NULL,
|
|
187
|
+
embedding vector NOT NULL,
|
|
188
|
+
source TEXT,
|
|
189
|
+
langchain_metadata JSONB,
|
|
190
|
+
docstore_doc_id UUID,
|
|
191
|
+
eventTime TIMESTAMPTZ
|
|
192
|
+
);
|
|
193
|
+
'''
|
|
194
|
+
self.execute_sql(sql)
|
|
195
|
+
|
|
196
|
+
self.grant_table_permissions(vectorstore_id, users)
|
sunholo/gcs/add_file.py
CHANGED
|
@@ -26,6 +26,10 @@ from ..utils.config import load_config_key
|
|
|
26
26
|
|
|
27
27
|
|
|
28
28
|
def handle_base64_image(base64_data, vector_name):
|
|
29
|
+
model = load_config_key("llm", vector_name, "vacConfig")
|
|
30
|
+
if model.startswith("openai"): # pass it to gpt directly
|
|
31
|
+
return base64_data, base64_data.split(",",1)
|
|
32
|
+
|
|
29
33
|
try:
|
|
30
34
|
header, encoded = base64_data.split(",", 1)
|
|
31
35
|
data = base64.b64decode(encoded)
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
Metadata-Version: 2.1
|
|
2
2
|
Name: sunholo
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.68.0
|
|
4
4
|
Summary: Large Language Model DevOps - a package to help deploy LLMs to the Cloud.
|
|
5
5
|
Home-page: https://github.com/sunholo-data/sunholo-py
|
|
6
|
-
Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.
|
|
6
|
+
Download-URL: https://github.com/sunholo-data/sunholo-py/archive/refs/tags/v0.68.0.tar.gz
|
|
7
7
|
Author: Holosun ApS
|
|
8
8
|
Author-email: multivac@sunholo.com
|
|
9
9
|
License: Apache License, Version 2.0
|
|
@@ -13,7 +13,7 @@ sunholo/agents/fastapi/base.py,sha256=clk76cHbUAvU0OYJrRfCWX_5f0ACbhDsIzYBhI3wyo
|
|
|
13
13
|
sunholo/agents/fastapi/qna_routes.py,sha256=DgK4Btu5XriOC1JaRQ4G_nWEjJfnQ0J5pyLanF6eF1g,3857
|
|
14
14
|
sunholo/agents/flask/__init__.py,sha256=uqfHNw2Ru3EJ4dJEcbp86h_lkquBQPMxZbjhV_xe3rs,72
|
|
15
15
|
sunholo/agents/flask/base.py,sha256=FgSaCODyoTtlstJtsqlLPScdgRUtv9_plxftdzHdVFo,809
|
|
16
|
-
sunholo/agents/flask/qna_routes.py,sha256=
|
|
16
|
+
sunholo/agents/flask/qna_routes.py,sha256=oDZzI0FllRD5GZI_C8EbKvvBSrgRlvmpwQc7lp54Krs,21926
|
|
17
17
|
sunholo/archive/__init__.py,sha256=qNHWm5rGPVOlxZBZCpA1wTYPbalizRT7f8X4rs2t290,31
|
|
18
18
|
sunholo/archive/archive.py,sha256=C-UhG5x-XtZ8VheQp92IYJqgD0V3NFQjniqlit94t18,1197
|
|
19
19
|
sunholo/auth/__init__.py,sha256=4owDjSaWYkbTlPK47UHTOC0gCWbZsqn4ZIEw5NWZTlg,28
|
|
@@ -44,10 +44,11 @@ sunholo/cli/sun_rich.py,sha256=UpMqeJ0C8i0pkue1AHnnyyX0bFJ9zZeJ7HBR6yhuA8A,54
|
|
|
44
44
|
sunholo/cli/swagger.py,sha256=absYKAU-7Yd2eiVNUY-g_WLl2zJfeRUNdWQ0oH8M_HM,1564
|
|
45
45
|
sunholo/components/__init__.py,sha256=IDoylb74zFKo6NIS3RQqUl0PDFBGVxM1dfUmO7OJ44U,176
|
|
46
46
|
sunholo/components/llm.py,sha256=T4we3tGmqUj4tPwxQr9M6AXv_BALqZV_dRSvINan-oU,10374
|
|
47
|
-
sunholo/components/retriever.py,sha256=
|
|
47
|
+
sunholo/components/retriever.py,sha256=jltG91N5r2P9RWKPW8A8tU3ilghciBczxapauW83Ir8,6377
|
|
48
48
|
sunholo/components/vectorstore.py,sha256=BxtMF_wX8Zrexr67P07OTSJPjucTewmcPM5OQwIXHPM,5630
|
|
49
49
|
sunholo/database/__init__.py,sha256=Zz0Shcq-CtStf9rJGIYB_Ybzb8rY_Q9mfSj-nviM490,241
|
|
50
|
-
sunholo/database/alloydb.py,sha256=
|
|
50
|
+
sunholo/database/alloydb.py,sha256=d9W0pbZB0jTVIGF5OVaQ6kXHo-X3-6e9NpWNmV5e9UY,10464
|
|
51
|
+
sunholo/database/alloydb_client.py,sha256=AYA0SSaBy-1XEfeZI97sMGehfrwnfbwZ8sE0exzI2E0,7254
|
|
51
52
|
sunholo/database/database.py,sha256=UDHkceiEvJmS3esQX2LYEjEMrHcogN_JHuJXoVWCH3M,7354
|
|
52
53
|
sunholo/database/lancedb.py,sha256=2rAbJVusMrm5TPtVTsUtmwn0z1iZ_wvbKhc6eyT6ClE,708
|
|
53
54
|
sunholo/database/static_dbs.py,sha256=aOyU3AJ-Dzz3qSNjbuN2293cfYw5PhkcQuQxdwPMJ4w,435
|
|
@@ -61,7 +62,7 @@ sunholo/database/sql/sb/setup.sql,sha256=CvoFvZQev2uWjmFa3aj3m3iuPFzAAJZ0S7Qi3L3
|
|
|
61
62
|
sunholo/embedder/__init__.py,sha256=sI4N_CqgEVcrMDxXgxKp1FsfsB4FpjoXgPGkl4N_u4I,44
|
|
62
63
|
sunholo/embedder/embed_chunk.py,sha256=P744zUQJgqrjILunzaqtTerB9AwoXFU6tXBtz4rjWgQ,6673
|
|
63
64
|
sunholo/gcs/__init__.py,sha256=DtVw_AZwQn-IguR5BJuIi2XJeF_FQXizhJikzRNrXiE,50
|
|
64
|
-
sunholo/gcs/add_file.py,sha256=
|
|
65
|
+
sunholo/gcs/add_file.py,sha256=y2s7ZZBiJD3pu1TqRAy0s1SRGvHvzJqSDaYD-MFFh1c,5717
|
|
65
66
|
sunholo/gcs/download_url.py,sha256=8XSEf8byfubqs5CMQeF_tn9wxqwUTq3n9mo5mLNIUTA,4801
|
|
66
67
|
sunholo/gcs/metadata.py,sha256=C9sMPsHsq1ETetdQCqB3EBs3Kws8b8QHS9L7ei_v5aw,891
|
|
67
68
|
sunholo/langfuse/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
@@ -105,9 +106,9 @@ sunholo/vertex/__init__.py,sha256=JvHcGFuv6R_nAhY2AdoqqhMpJ5ugeWPZ_svGhWrObBk,13
|
|
|
105
106
|
sunholo/vertex/init.py,sha256=JDMUaBRdednzbKF-5p33qqLit2LMsvgvWW-NRz0AqO0,1801
|
|
106
107
|
sunholo/vertex/memory_tools.py,sha256=8F1iTWnqEK9mX4W5RzCVKIjydIcNp6OFxjn_dtQ3GXo,5379
|
|
107
108
|
sunholo/vertex/safety.py,sha256=3meAX0HyGZYrH7rXPUAHxtI_3w_zoy_RX7Shtkoa660,1275
|
|
108
|
-
sunholo-0.
|
|
109
|
-
sunholo-0.
|
|
110
|
-
sunholo-0.
|
|
111
|
-
sunholo-0.
|
|
112
|
-
sunholo-0.
|
|
113
|
-
sunholo-0.
|
|
109
|
+
sunholo-0.68.0.dist-info/LICENSE.txt,sha256=SdE3QjnD3GEmqqg9EX3TM9f7WmtOzqS1KJve8rhbYmU,11345
|
|
110
|
+
sunholo-0.68.0.dist-info/METADATA,sha256=G2k3HIbR1aD0HK5mrD_Rr8E_jusewG5z8dgW_DCsmiA,6155
|
|
111
|
+
sunholo-0.68.0.dist-info/WHEEL,sha256=mguMlWGMX-VHnMpKOjjQidIo1ssRlCFu4a4mBpz1s2M,91
|
|
112
|
+
sunholo-0.68.0.dist-info/entry_points.txt,sha256=bZuN5AIHingMPt4Ro1b_T-FnQvZ3teBes-3OyO0asl4,49
|
|
113
|
+
sunholo-0.68.0.dist-info/top_level.txt,sha256=wt5tadn5--5JrZsjJz2LceoUvcrIvxjHJe-RxuudxAk,8
|
|
114
|
+
sunholo-0.68.0.dist-info/RECORD,,
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|