kobai-sdk 0.2.8rc11__py3-none-any.whl → 0.2.8rc12__py3-none-any.whl

kobai/ai_rag.py

@@ -143,7 +143,7 @@ def __generate_sentences_from_questions(tc: AIContext, debug):
     ss.sql(full_sql)
 
 
-def encode_to_delta_local(tc: AIContext, st_model: Union[SentenceTransformer, Embeddings], replica_schema=None):
+def encode_to_delta_local(tc: AIContext, st_model: Union[SentenceTransformer, Embeddings], replica_schema=None, batch_size=100000):
     """
     Encode Semantic Data to Vectors in Delta Table
 
@@ -163,7 +163,7 @@ def encode_to_delta_local(tc: AIContext, st_model: Union[SentenceTransformer, Em
     sentences_df = ss.sql(sentences_sql)
 
     num_records = sentences_df.count()
-    query_batch_size = 100000
+    query_batch_size = batch_size
 
     for x in range(0, num_records, query_batch_size):
         print(f"Running Batch Starting at {x}")
@@ -174,9 +174,11 @@ def encode_to_delta_local(tc: AIContext, st_model: Union[SentenceTransformer, Em
 
         if isinstance(st_model, SentenceTransformer):
             vector_list = st_model.encode(
-                content_list, normalize_embeddings=True, show_progress_bar=True)
+                content_list, normalize_embeddings=True, show_progress_bar=True).tolist()
         else:
             vector_list = st_model.embed_documents(content_list)
+            for i, v in enumerate(vector_list):
+                vector_list[i] = [float(x) for x in v]
         #vector_list = st_model.encode(
         #    content_list, normalize_embeddings=True, show_progress_bar=True)
 
@@ -185,7 +187,7 @@ def encode_to_delta_local(tc: AIContext, st_model: Union[SentenceTransformer, Em
             StructField("vector", ArrayType(FloatType()), False)
         ])
 
-        updated_list = [[r[0], r[1].tolist()]
+        updated_list = [[r[0], r[1]]
                         for r in zip(id_list, vector_list)]
         updated_df = ss.createDataFrame(updated_list, schema_v)
 

kobai/tenant_client.py

@@ -451,7 +451,7 @@ class TenantClient:
         """
         ai_rag.generate_sentences(self.get_ai_context(), replica_schema=replica_schema, concept_white_list=concept_white_list, use_questions=use_questions, debug=debug)
 
-    def rag_encode_to_delta_local(self, st_model: Union[SentenceTransformer, Embeddings], replica_schema=None):
+    def rag_encode_to_delta_local(self, st_model: Union[SentenceTransformer, Embeddings], replica_schema=None, batch_size=100000):
         """
         Encode Semantic Data to Vectors in Delta Table
 
@@ -459,7 +459,7 @@ class TenantClient:
         st_model (SentenceTransformer): A sentence_transformers model to use for encoding.
         replica_schema (str) OPTIONAL: An alternate schema (catalog.database) to create the Delta table. Useful when the base Kobai schema is not on a Unity Catalog.
         """
-        ai_rag.encode_to_delta_local(self.get_ai_context(), st_model=st_model, replica_schema=replica_schema)
+        ai_rag.encode_to_delta_local(self.get_ai_context(), st_model=st_model, replica_schema=replica_schema, batch_size=batch_size)
 
     def rag_delta(self, emb_model: Union[SentenceTransformer, Embeddings], chat_model: BaseChatModel, question, k=5, replica_schema=None):
         """

{kobai_sdk-0.2.8rc11.dist-info → kobai_sdk-0.2.8rc12.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.2
 Name: kobai-sdk
-Version: 0.2.8rc11
+Version: 0.2.8rc12
 Summary: A package that enables interaction with a Kobai tenant.
 Author-email: Ryan Oattes <ryan@kobai.io>
 License:                                  Apache License

{kobai_sdk-0.2.8rc11.dist-info → kobai_sdk-0.2.8rc12.dist-info}/RECORD

@@ -1,13 +1,13 @@
 kobai/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 kobai/ai_query.py,sha256=d0WOPKtQ-bI_zW3-_6guEJX0t55OcxdXIdgXaD-zKK0,9413
-kobai/ai_rag.py,sha256=Gqj_CF2lLjMc0CcxquOAgXhiuY6pKwyJumR2NK7QdOo,14559
+kobai/ai_rag.py,sha256=hc8M7M9azY-tFovsXdrjHJTmemFr8L74X2dy0j-XUoY,14685
 kobai/databricks_client.py,sha256=fyqqMly2Qm0r1AHWsQjkYeNsDdH0G1JSgTkF9KJ55qA,2118
 kobai/demo_tenant_client.py,sha256=wlNc-bdI2wotRXo8ppUOalv4hYdBlek_WzJNARZV-AE,9293
 kobai/spark_client.py,sha256=opM_F-4Ut5Hq5zZjWMuLvUps9sDULvyPNZHXGL8dW1k,776
 kobai/tenant_api.py,sha256=9U6UbxpaAb-kpbuADXx3kbkNKaOzYy0I-GGwbpiCCOk,4212
-kobai/tenant_client.py,sha256=ktw75dmmX95gD6qIkG-mAmdo48RkF96SgmpRmfDWblo,39422
-kobai_sdk-0.2.8rc11.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
-kobai_sdk-0.2.8rc11.dist-info/METADATA,sha256=jyrtCYbqTYKrHMHBOmWUrDfkK3NFqtqxmQyTW-vZ05A,19205
-kobai_sdk-0.2.8rc11.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
-kobai_sdk-0.2.8rc11.dist-info/top_level.txt,sha256=ns1El3BrTTHKvoAgU1XtiSaVIudYeCXbEEUVY8HFDZ4,6
-kobai_sdk-0.2.8rc11.dist-info/RECORD,,
+kobai/tenant_client.py,sha256=OUHQMmqWlezenkmMJH9p4wOz5CsAntmO25v5vWdepVE,39464
+kobai_sdk-0.2.8rc12.dist-info/LICENSE,sha256=QwcOLU5TJoTeUhuIXzhdCEEDDvorGiC6-3YTOl4TecE,11356
+kobai_sdk-0.2.8rc12.dist-info/METADATA,sha256=vrkBxo0XcZ1uG4lPHs-XfL38rxu-az55yw4NuJ_v5rk,19205
+kobai_sdk-0.2.8rc12.dist-info/WHEEL,sha256=In9FTNxeP60KnTkGw7wk6mJPYd_dQSjEZmXdBdMCI-8,91
+kobai_sdk-0.2.8rc12.dist-info/top_level.txt,sha256=ns1El3BrTTHKvoAgU1XtiSaVIudYeCXbEEUVY8HFDZ4,6
+kobai_sdk-0.2.8rc12.dist-info/RECORD,,