kolzchut-ragbot 1.4.1__py3-none-any.whl → 1.5.0__py3-none-any.whl

kolzchut_ragbot/model.py

@@ -1,179 +1,182 @@
-import datetime
-import logging
-import os
-from .Document import factory as definitions_factory
-
-definitions_singleton = definitions_factory()
-EMBEDDING_INDEX = os.getenv("ES_EMBEDDING_INDEX", "embeddings")
-semantic_models = definitions_singleton.models.keys()
-
-
-def index_from_page_id(page_id: int):
-    """
-    Generates an index name based on the page ID.
-
-    Args:
-        page_id (int): The ID of the page.
-
-    Returns:
-        str: The generated index name.
-    """
-    index_postfix = round(page_id / 1000)
-    return EMBEDDING_INDEX + "_" + str(index_postfix)
-
-def create_mapping():
-    """
-    Creates a mapping for the model in Elasticsearch.
-    """
-    vector_fields = {f"{name}_{semantic_model}_vectors": {"type": "dense_vector", "dims": 1024}
-                     for name, semantic_model in definitions_singleton.models.items()}
-
-    data_fields = {}
-    for field in definitions_singleton.saved_fields.keys():
-        field_type = definitions_singleton.saved_fields[field]
-        field_mapping = {"type": field_type}
-        if field_type == "date":
-            field_mapping["format"] = "yyyyMMddHHmmss"
-        data_fields[f"{field}"] = field_mapping
-
-    mappings = {
-        "properties": {
-            "last_update": {
-                "type": "date",
-            },
-            **vector_fields,
-            **data_fields,
-        }
-    }
-    return mappings
-
-class Model:
-    """
-    Represents the model for creating, updating, and searching documents in Elasticsearch.
-
-    Attributes:
-        custom_result_selection_function (callable): A custom function for selecting search results.
-        es_client: The Elasticsearch client instance.
-
-    Methods:
-        create_index():
-            Creates an index for the model in Elasticsearch.
-
-        create_or_update_documents(paragraphs_dicts: list[dict], update=False):
-            Creates or updates documents in the Elasticsearch index.
-
-        search(embedded_search: dict[str, list[float]], size=50) -> dict[str, list[dict]]:
-            Searches for similar documents using cosine similarity.
-    """
-
-    custom_result_selection_function = None
-
-    def __init__(self, es_client, custom_result_selection_function=None):
-        """
-        Initializes the Model instance.
-
-        Args:
-            es_client: The Elasticsearch client instance.
-            custom_result_selection_function (callable, optional): A custom function for selecting search results.
-        """
-        self.es_client = es_client
-        if custom_result_selection_function is not None:
-            self.custom_result_selection_function = custom_result_selection_function
-
-    def create_index(self):
-        """
-        Creates an index for the model in Elasticsearch.
-        """
-        mapping = create_mapping()
-        if not self.es_client.indices.exists(index=os.getenv("ES_EMBEDDING_INDEX")):
-            self.es_client.indices.create(
-                index=os.getenv("ES_EMBEDDING_INDEX"),
-                mappings=mapping)
-
-    def create_or_update_documents(self, paragraphs_dicts: list[dict], update=False):
-        """
-        Creates or updates documents in the Elasticsearch index.
-
-        Args:
-            paragraphs_dicts (list[dict]): A list of dictionaries representing the paragraphs to be indexed.
-            update (bool, optional): Whether to update existing documents. Default is False.
-        """
-
-        identifier = definitions_singleton.identifier
-        print(f"Creating or updating documents in the index, {len(paragraphs_dicts)} paragraphs\n")
-        # Identify the doc from the first paragraph - all paragraphs should have the same doc_id
-        doc_id = paragraphs_dicts[0][identifier]
-        index = index_from_page_id(int(doc_id))
-
-        if update:
-            try:
-                query = {
-                    "query": {
-                        "match": {
-                            f"{identifier}": doc_id
-                        }
-                    }
-                }
-                self.es_client.delete_by_query(index=f"{EMBEDDING_INDEX}*", body=query)
-
-            except Exception as e:
-                logging.error(f"Error while searching for existing document: {e}")
-
-        for i, doc_dict in enumerate(paragraphs_dicts):
-            print(f"saving paragraph {i}")
-            doc = {
-                "last_update": datetime.datetime.now(),
-                **doc_dict
-            }
-
-            self.es_client.index(index=index, body=doc)
-
-    def search(self, embedded_search: dict[str, list[float]], size=50) -> dict[str, list[dict]]:
-        """
-        Searches for similar documents using cosine similarity.
-
-        Args:
-            embedded_search (dict[str, list[float]]): A dictionary containing the embedded search vectors.
-            size (int, optional): The number of search results to return. Default is 50.
-
-        Returns:
-            dict[str, list[dict]]: A dictionary containing the search results.
-        """
-        results = {}
-        for semantic_model, field in definitions_singleton.models.items():
-            results[field] = [] if field not in results.keys() else results[field]
-            body = {
-                "script_score": {
-                    "query": {
-                        "exists": {
-                            "field": f'{field}_{semantic_model}_vectors'
-                        }
-                    },
-                    "script": {
-                        "source": f"cosineSimilarity(params.query_vector, '{field}_{semantic_model}_vectors') + 1.0",
-                        "params": {
-                            "query_vector": embedded_search[semantic_model]
-                        }
-                    }
-                }
-            }
-            print(f"Searching for {field} using {semantic_model} on index {EMBEDDING_INDEX}\n")
-            field_results = self.es_client.search(
-                index=EMBEDDING_INDEX + "*",
-                body={
-                    "size": size,
-                    "query": body
-                })
-            results[field] = results[field] + field_results["hits"]["hits"]
-
-        return results
-
-
-model = None
-
-
-def es_client_factory(es_client) -> Model:
-    global model
-    if model is None:
-        model = Model(es_client)
-    return model
+import datetime
+import logging
+import os
+from .Document import factory as definitions_factory
+
+definitions_singleton = definitions_factory()
+EMBEDDING_INDEX = os.getenv("ES_EMBEDDING_INDEX", "embeddings")
+semantic_models = definitions_singleton.models.keys()
+
+
+def index_from_page_id(page_id: int):
+    """
+    Generates an index name based on the page ID.
+
+    Args:
+        page_id (int): The ID of the page.
+
+    Returns:
+        str: The generated index name.
+    """
+    index_postfix = round(page_id / 1000)
+    return EMBEDDING_INDEX + "_" + str(index_postfix)
+
+
+def create_mapping():
+    """
+    Creates a mapping for the model in Elasticsearch.
+    """
+    vector_fields = {f'{semantic_model}_{name}_vectors': {"type": "dense_vector", "dims": 1024}
+                     for name, semantic_model in definitions_singleton.models.items()}
+
+    data_fields = {}
+    for field in definitions_singleton.saved_fields.keys():
+        field_type = definitions_singleton.saved_fields[field]
+        field_mapping = {"type": field_type}
+        if field_type == "date":
+            field_mapping["format"] = "yyyyMMddHHmmss"
+        data_fields[f"{field}"] = field_mapping
+
+    mappings = {
+        "properties": {
+            "last_update": {
+                "type": "date",
+            },
+            **vector_fields,
+            **data_fields,
+        }
+    }
+    return mappings
+
+
+class Model:
+    """
+    Represents the model for creating, updating, and searching documents in Elasticsearch.
+
+    Attributes:
+        custom_result_selection_function (callable): A custom function for selecting search results.
+        es_client: The Elasticsearch client instance.
+
+    Methods:
+        create_index():
+            Creates an index for the model in Elasticsearch.
+
+        create_or_update_documents(paragraphs_dicts: list[dict], update=False):
+            Creates or updates documents in the Elasticsearch index.
+
+        search(embedded_search: dict[str, list[float]], size=50) -> dict[str, list[dict]]:
+            Searches for similar documents using cosine similarity.
+    """
+
+    custom_result_selection_function = None
+
+    def __init__(self, es_client, custom_result_selection_function=None):
+        """
+        Initializes the Model instance.
+
+        Args:
+            es_client: The Elasticsearch client instance.
+            custom_result_selection_function (callable, optional): A custom function for selecting search results.
+        """
+        self.es_client = es_client
+        if custom_result_selection_function is not None:
+            self.custom_result_selection_function = custom_result_selection_function
+
+    def create_index(self, index_name):
+        """
+        Creates an index for the model in Elasticsearch.
+        """
+        mapping = create_mapping()
+        if not self.es_client.indices.exists(index=index_name):
+            self.es_client.indices.create(
+                index=index_name,
+                mappings=mapping
+            )
+
+    def create_or_update_documents(self, paragraphs_dicts: list[dict], update=False):
+        """
+        Creates or updates documents in the Elasticsearch index.
+
+        Args:
+            paragraphs_dicts (list[dict]): A list of dictionaries representing the paragraphs to be indexed.
+            update (bool, optional): Whether to update existing documents. Default is False.
+        """
+
+        identifier = definitions_singleton.identifier
+        print(f"Creating or updating documents in the index, {len(paragraphs_dicts)} paragraphs\n")
+        # Identify the doc from the first paragraph - all paragraphs should have the same doc_id
+        doc_id = paragraphs_dicts[0][identifier]
+        index = index_from_page_id(int(doc_id))
+
+        if update:
+            try:
+                query = {
+                    "query": {
+                        "match": {
+                            f"{identifier}": doc_id
+                        }
+                    }
+                }
+                self.es_client.delete_by_query(index=f"{EMBEDDING_INDEX}*", body=query)
+
+            except Exception as e:
+                logging.error(f"Error while searching for existing document: {e}")
+        self.create_index(index)
+        for i, doc_dict in enumerate(paragraphs_dicts):
+            print(f"saving paragraph {i + 1} / {len(paragraphs_dicts)}")
+            doc = {
+                "last_update": datetime.datetime.now(),
+                **doc_dict
+            }
+
+            self.es_client.index(index=index, body=doc)
+
+    def search(self, embedded_search: dict[str, list[float]], size=50) -> dict[str, list[dict]]:
+        """
+        Searches for similar documents using cosine similarity.
+
+        Args:
+            embedded_search (dict[str, list[float]]): A dictionary containing the embedded search vectors.
+            size (int, optional): The number of search results to return. Default is 50.
+
+        Returns:
+            dict[str, list[dict]]: A dictionary containing the search results.
+        """
+        results = {}
+        for semantic_model, field in definitions_singleton.models.items():
+            results[field] = [] if field not in results.keys() else results[field]
+            body = {
+                "script_score": {
+                    "query": {
+                        "exists": {
+                            "field": f'{field}_{semantic_model}_vectors'
+                        }
+                    },
+                    "script": {
+                        "source": f"cosineSimilarity(params.query_vector, '{field}_{semantic_model}_vectors') + 1.0",
+                        "params": {
+                            "query_vector": embedded_search[semantic_model]
+                        }
+                    }
+                }
+            }
+            print(f"Searching for {field} using {semantic_model} on index {EMBEDDING_INDEX}\n")
+            field_results = self.es_client.search(
+                index=EMBEDDING_INDEX + "*",
+                body={
+                    "size": size,
+                    "query": body
+                })
+            results[field] = results[field] + field_results["hits"]["hits"]
+
+        return results
+
+
+model = None
+
+
+def es_client_factory(es_client) -> Model:
+    global model
+    if model is None:
+        model = Model(es_client)
+    return model

kolzchut_ragbot-1.5.0.dist-info/METADATA

@@ -0,0 +1,76 @@
+Metadata-Version: 2.4
+Name: kolzchut-ragbot
+Version: 1.5.0
+Summary: A search engine using machine learning models and Elasticsearch for advanced document retrieval.
+Home-page: https://github.com/shmuelrob/rag-bot
+Author: Shmuel Robinov
+Author-email: shmuel_robinov@webiks.com
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: MIT License
+Classifier: Operating System :: OS Independent
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+Requires-Dist: elasticsearch==8.17.1
+Requires-Dist: sentence-transformers==3.4.1
+Requires-Dist: torch==2.6.0
+Requires-Dist: transformers==4.48.3
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# kolzchut-ragbot
+
+## Overview
+
+This project is a search engine that uses machine learning models and Elasticsearch to provide advanced document retrieval.
+You can use [kolzchut-ragbot](https://github.com/shmuelrob/rag-bot) to demonstrate the engine's document retrieval abilities.
+
+## Features
+
+- Document representation and validation
+- Document embedding and indexing in Elasticsearch
+- Advanced search using machine learning model
+- Integration with LLM (Large Language Model) client for query answering
+
+## Installation
+
+### From PyPI
+
+```bash
+pip install kolzchut-ragbot
+```
+
+### From Source
+
+1. Clone the repository:
+   
+   ```bash
+   git clone https://github.com/shmuelrob/rag-bot.git
+   cd rag-bot
+   ```
+
+2. Create a virtual environment and activate it:  
+
+   ```bash
+   python -m venv venv
+   source venv/bin/activate  # On Windows use: venv\Scripts\activate
+   ```
+
+3. Install the required dependencies:  
+
+   ```bash
+   pip install -r requirements.txt
+   ```
+
+## Configuration
+
+Set the following environment variables:
+
+- `ES_EMBEDDING_INDEX`: The name of the Elasticsearch index for embeddings.
+- `TOKENIZER_LOCATION`: The location of the tokenizer model.

kolzchut_ragbot-1.5.0.dist-info/RECORD

@@ -0,0 +1,11 @@
+kolzchut_ragbot/Document.py,sha256=5OyBBTZyAJFM_1Pjs3SUC-_s5zEJ5U6wjhw12_FFkdE,3621
+kolzchut_ragbot/IntegrateService.py,sha256=rcwUY2RkclCY3l8BGAmNbstdxhxwhLO9oA8BofqLyts,96
+kolzchut_ragbot/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+kolzchut_ragbot/config.py,sha256=uILFvgn9W92-NRaKXYtaoQXpn3KOWKK8SZYRsIAa5Yw,133
+kolzchut_ragbot/engine.py,sha256=tULiiCdqLoXydxx7VrZKChDgbe4ygSoEft-k76j3_t0,10158
+kolzchut_ragbot/llm_client.py,sha256=Frp7CL0OIlQA6ltohrGWedI6uD6MpGg6TbpZTBE0qIo,341
+kolzchut_ragbot/model.py,sha256=HCi3r4YztPknnbgTOA7I-GVaqxn8CzrTeLFkEg-7fg0,6320
+kolzchut_ragbot-1.5.0.dist-info/METADATA,sha256=nzWcZsfn23nVWANcWMGoQvQXJ-idymUKg0ZheMbWp1c,1999
+kolzchut_ragbot-1.5.0.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+kolzchut_ragbot-1.5.0.dist-info/top_level.txt,sha256=NTZoY4GGw3v_7jm0MgcdHw8simoZ78PsR7Meqmkgd_Q,16
+kolzchut_ragbot-1.5.0.dist-info/RECORD,,

{kolzchut_ragbot-1.4.1.dist-info → kolzchut_ragbot-1.5.0.dist-info}/WHEEL

@@ -1,5 +1,5 @@
 Wheel-Version: 1.0
-Generator: setuptools (78.1.0)
+Generator: setuptools (80.9.0)
 Root-Is-Purelib: true
 Tag: py3-none-any
 

kolzchut_ragbot-1.4.1.dist-info/METADATA

@@ -1,67 +0,0 @@
-Metadata-Version: 2.4
-Name: kolzchut-ragbot
-Version: 1.4.1
-Summary: A search engine using machine learning models and Elasticsearch for advanced document retrieval.
-Home-page: https://github.com/shmuelrob/rag-bot
-Author: Shmuel Robinov
-Author-email: shmuel_robinov@webiks.com
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: MIT License
-Classifier: Operating System :: OS Independent
-Requires-Python: >=3.10
-Description-Content-Type: text/markdown
-Requires-Dist: elasticsearch==8.17.1
-Requires-Dist: sentence-transformers==3.4.1
-Requires-Dist: torch==2.6.0
-Requires-Dist: transformers==4.48.3
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# **Webiks-Hebrew-RAGbot**
-
-## **Overview**
-
-This project is a search engine that uses machine learning models and Elasticsearch to provide advanced document retrieval.
-You can use [Webiks-Hebrew-RAGbot-Demo](https://github.com/NNLP-IL/Webiks-Hebrew-RAGbot-Demo) to demonstrate the engine's document retrieval abilities
-
-## **Features**
-
-Document representation and validation
-Document embedding and indexing in Elasticsearch
-Advanced search using machine learning model
-Integration with LLM (Large Language Model) client for query answering
-
-## **Installation**
-
-1.  Clone the repository:
-   
-`git clone https://github.com/NNLP-IL/Webiks-Hebrew-RAGbot.git`
-
-`cd Webiks-Hebrew-RAGbot`
-
-2.  Create a virtual environment and activate it:  
-
-`python -m venv venv`
-
-`source venv/bin/activate`
-
-On Windows use `\venv\\Scripts\\activate\`
-
-3.  Install the required dependencies:  
-
-`pip install -r requirements.txt`
-
-## **Configuration**
-
-Set the following environment variables:  
-
-ES\_EMBEDDING\_INDEX: The name of the Elasticsearch index for embeddings.
-
-TOKENIZER\_LOCATION: The location of the tokenizer model.

kolzchut_ragbot-1.4.1.dist-info/RECORD

@@ -1,11 +0,0 @@
-kolzchut_ragbot/Document.py,sha256=ySawnD06HA0zHjHp4Y_CPjMMZqLp8onaEgd1dGP5sbs,3722
-kolzchut_ragbot/IntegrateService.py,sha256=CqB9vW6W5oj6Ig3crEa6hXqwro21z97UaG9ngxFTzYs,100
-kolzchut_ragbot/__init__.py,sha256=KKAc2xjCl5Aui2Cj0FWyvJ51nmnFv7MspLMqOYb-QHA,26
-kolzchut_ragbot/config.py,sha256=pcKVJVJ8P2YximjTrmVlrocHXSmzmNu_DFzNoPLa22E,138
-kolzchut_ragbot/engine.py,sha256=V8WUWyqvBWbGt-rRRf8G6BEyD-4GjsmtJrxBb6aPon8,10154
-kolzchut_ragbot/llm_client.py,sha256=q_cUZq645P7i1PliYzpJRTWlsoSECVIhE-y9wU5eRtQ,352
-kolzchut_ragbot/model.py,sha256=Nd3_0KXMfyajQTyreKpkb2hsqpkA6_P994laoRKF2iU,6452
-kolzchut_ragbot-1.4.1.dist-info/METADATA,sha256=oI7PoQRz2ddDIRUtVmVAa0FF8YSlAjQFPzLhC0lq81A,2024
-kolzchut_ragbot-1.4.1.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91
-kolzchut_ragbot-1.4.1.dist-info/top_level.txt,sha256=NTZoY4GGw3v_7jm0MgcdHw8simoZ78PsR7Meqmkgd_Q,16
-kolzchut_ragbot-1.4.1.dist-info/RECORD,,