kolzchut-ragbot 1.4.0__py3-none-any.whl → 1.4.2__py3-none-any.whl

kolzchut_ragbot/model.py

@@ -22,6 +22,33 @@ def index_from_page_id(page_id: int):
     return EMBEDDING_INDEX + "_" + str(index_postfix)
 
 
+def create_mapping():
+    """
+    Creates a mapping for the model in Elasticsearch.
+    """
+    vector_fields = {f'{semantic_model}_{name}_vectors': {"type": "dense_vector", "dims": 1024}
+                     for name, semantic_model in definitions_singleton.models.items()}
+
+    data_fields = {}
+    for field in definitions_singleton.saved_fields.keys():
+        field_type = definitions_singleton.saved_fields[field]
+        field_mapping = {"type": field_type}
+        if field_type == "date":
+            field_mapping["format"] = "yyyyMMddHHmmss"
+        data_fields[f"{field}"] = field_mapping
+
+    mappings = {
+        "properties": {
+            "last_update": {
+                "type": "date",
+            },
+            **vector_fields,
+            **data_fields,
+        }
+    }
+    return mappings
+
+
 class Model:
     """
     Represents the model for creating, updating, and searching documents in Elasticsearch.
@@ -55,27 +82,16 @@ class Model:
         if custom_result_selection_function is not None:
             self.custom_result_selection_function = custom_result_selection_function
 
-    def create_index(self):
+    def create_index(self, index_name):
         """
         Creates an index for the model in Elasticsearch.
         """
-        if not self.es_client.indices.exists(index=os.getenv("ES_EMBEDDING_INDEX")):
-            vector_fields = {f"{name}_{semantic_model}_vectors": {"type": "dense_vector", "dims": 1024}
-                             for name, semantic_model in definitions_singleton.models.items()}
-            data_fields = {f"{field}": {"type": definitions_singleton.saved_fields[field]}
-                           for field in definitions_singleton.saved_fields.keys()}
-
+        mapping = create_mapping()
+        if not self.es_client.indices.exists(index=index_name):
             self.es_client.indices.create(
-                index=os.getenv("ES_EMBEDDING_INDEX"),
-                mappings={
-                    "properties": {
-                        "last_update": {
-                            "type": "date"
-                        },
-                        **vector_fields,
-                        **data_fields,
-                    }
-                })
+                index=index_name,
+                mappings=mapping
+            )
 
     def create_or_update_documents(self, paragraphs_dicts: list[dict], update=False):
         """
@@ -105,9 +121,9 @@ class Model:
 
             except Exception as e:
                 logging.error(f"Error while searching for existing document: {e}")
-
+        self.create_index(index)
         for i, doc_dict in enumerate(paragraphs_dicts):
-            print(f"saving paragraph {i}")
+            print(f"saving paragraph {i + 1} / {len(paragraphs_dicts)}")
             doc = {
                 "last_update": datetime.datetime.now(),
                 **doc_dict

{kolzchut_ragbot-1.4.0.dist-info → kolzchut_ragbot-1.4.2.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: kolzchut-ragbot
-Version: 1.4.0
+Version: 1.4.2
 Summary: A search engine using machine learning models and Elasticsearch for advanced document retrieval.
 Home-page: https://github.com/shmuelrob/rag-bot
 Author: Shmuel Robinov

{kolzchut_ragbot-1.4.0.dist-info → kolzchut_ragbot-1.4.2.dist-info}/RECORD

@@ -4,8 +4,8 @@ kolzchut_ragbot/__init__.py,sha256=KKAc2xjCl5Aui2Cj0FWyvJ51nmnFv7MspLMqOYb-QHA,2
 kolzchut_ragbot/config.py,sha256=pcKVJVJ8P2YximjTrmVlrocHXSmzmNu_DFzNoPLa22E,138
 kolzchut_ragbot/engine.py,sha256=V8WUWyqvBWbGt-rRRf8G6BEyD-4GjsmtJrxBb6aPon8,10154
 kolzchut_ragbot/llm_client.py,sha256=q_cUZq645P7i1PliYzpJRTWlsoSECVIhE-y9wU5eRtQ,352
-kolzchut_ragbot/model.py,sha256=OfFFNj429IUUObOpjDB08ubZo6io45sOaoxkyEU68d4,6228
-kolzchut_ragbot-1.4.0.dist-info/METADATA,sha256=PGV1l69-kxkDMjtVPYrXudwLyag7cJvEZtUi1Tgo6CA,2024
-kolzchut_ragbot-1.4.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kolzchut_ragbot-1.4.0.dist-info/top_level.txt,sha256=NTZoY4GGw3v_7jm0MgcdHw8simoZ78PsR7Meqmkgd_Q,16
-kolzchut_ragbot-1.4.0.dist-info/RECORD,,
+kolzchut_ragbot/model.py,sha256=M7i9B-zzwa-ATblY-5c7gmbkOXKwS8wWmYMP8l0HE40,6502
+kolzchut_ragbot-1.4.2.dist-info/METADATA,sha256=YctMSApfbXBBvx7d0APy28lDWmulO3cfHJKjKOZ_LYQ,2024
+kolzchut_ragbot-1.4.2.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
+kolzchut_ragbot-1.4.2.dist-info/top_level.txt,sha256=NTZoY4GGw3v_7jm0MgcdHw8simoZ78PsR7Meqmkgd_Q,16
+kolzchut_ragbot-1.4.2.dist-info/RECORD,,