gllm-datastore-binary 0.0.15__cp311-cp311-win_amd64.whl

gllm_datastore/vector_data_store/elasticsearch_vector_data_store.pyi

@@ -0,0 +1,140 @@
+from _typeshed import Incomplete
+from gllm_core.schema import Chunk
+from gllm_datastore.constants import DEFAULT_REQUEST_TIMEOUT as DEFAULT_REQUEST_TIMEOUT, DEFAULT_TOP_K as DEFAULT_TOP_K
+from gllm_datastore.utils.converter import from_langchain as from_langchain, to_langchain as to_langchain
+from gllm_datastore.vector_data_store.vector_data_store import BaseVectorDataStore as BaseVectorDataStore
+from langchain_core.embeddings import Embeddings as Embeddings
+from typing import Any
+
+class ElasticsearchVectorDataStore(BaseVectorDataStore):
+    """DataStore for interacting with Elasticsearch.
+
+    This class provides methods for executing queries and retrieving documents
+    from Elasticsearch. It relies on the LangChain's ElasticsearchStore  for
+    vector operations and the underlying Elasticsearch client management.
+
+    Attributes:
+        store (ElasticsearchStore): The ElasticsearchStore instance for vector operations.
+        index_name (str): The name of the Elasticsearch index.
+        logger (Logger): The logger object.
+    """
+    index_name: Incomplete
+    store: Incomplete
+    logger: Incomplete
+    def __init__(self, index_name: str, embedding: Embeddings | None = None, connection: Any | None = None, url: str | None = None, cloud_id: str | None = None, user: str | None = None, api_key: str | None = None, password: str | None = None, vector_query_field: str = 'vector', query_field: str = 'text', distance_strategy: str | None = None, strategy: Any | None = None, request_timeout: int = ...) -> None:
+        '''Initializes an instance of the ElasticsearchVectorDataStore class.
+
+        Args:
+            index_name (str): The name of the Elasticsearch index.
+            embedding (Embeddings | None, optional): The Embeddings object for vector operations. Defaults to None.
+            connection (Any | None, optional): The Elasticsearch connection object. Defaults to None.
+            url (str | None, optional): The URL of the Elasticsearch server. Defaults to None.
+            cloud_id (str | None, optional): The cloud ID of the Elasticsearch cluster. Defaults to None.
+            user (str | None, optional): The username for authentication. Defaults to None.
+            api_key (str | None, optional): The API key for authentication. Defaults to None.
+            password (str | None, optional): The password for authentication. Defaults to None.
+            vector_query_field (str, optional): The field name for vector queries. Defaults to "vector".
+            query_field (str, optional): The field name for text queries. Defaults to "text".
+            distance_strategy (str | None, optional): The distance strategy for retrieval. Defaults to None.
+            strategy (Any | None, optional): The retrieval strategy for retrieval. Defaults to None, in which case
+                DenseVectorStrategy() is used.
+            request_timeout (int, optional): The request timeout. Defaults to DEFAULT_REQUEST_TIMEOUT.
+        '''
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Queries the Elasticsearch data store.
+
+        Args:
+            query (str): The query string.
+            top_k (int, optional): The number of top results to retrieve. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional retrieval parameters. Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of Chunk objects representing the retrieved documents.
+        """
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Queries the data store by ID and returns a list of Chunk objects.
+
+        Args:
+            id_: The ID of the document to query.
+
+        Returns:
+            A list of Chunk objects representing the queried documents.
+
+        Note:
+            This method not implement yet. Because the ElasticsearchStore
+            still not implement the get_by_ids method yet.
+        """
+    async def autocomplete(self, query: str, field: str, size: int = 20, fuzzy_tolerance: int = 1, min_prefix_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Provides suggestions based on a prefix query for a specific field.
+
+        Args:
+            query (str): The query string.
+            field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            fuzzy_tolerance (int, optional): The level of fuzziness for suggestions. Defaults to 1.
+            min_prefix_length (int, optional): The minimum prefix length to trigger fuzzy matching. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def autosuggest(self, query: str, search_fields: list[str], autocomplete_field: str, size: int = 20, min_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Generates suggestions across multiple fields using a multi_match query to broaden the search criteria.
+
+        Args:
+            query (str): The query string.
+            search_fields (list[str]): The fields to search for.
+            autocomplete_field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            min_length (int, optional): The minimum length of the query. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def shingles(self, query: str, field: str, size: int = 20, min_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Searches using shingles for prefix and fuzzy matching.
+
+        Args:
+            query (str): The query string.
+            field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            min_length (int, optional): The minimum length of the query. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def add_chunks(self, chunk: Chunk | list[Chunk], **kwargs: Any) -> list[str]:
+        """Adds a chunk or a list of chunks to the data store.
+
+        Args:
+            chunk (Chunk | list[Chunk]): The chunk or list of chunks to add.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added chunks.
+        """
+    async def add_embeddings(self, text_embeddings: list[tuple[str, list[float]]], metadatas: list[dict] | None = None, ids: list[str] | None = None, **kwargs) -> list[str]:
+        """Adds text embeddings to the data store.
+
+        Args:
+            text_embeddings (list[tuple[str, list[float]]]): Pairs of string and embedding to add to the store.
+            metadatas (list[dict], optional): Optional list of metadatas associated with the texts. Defaults to None.
+            ids (list[str], optional): Optional list of unique IDs. Defaults to None.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added embeddings.
+        """
+    async def delete_chunks(self, query: dict[str, Any], **kwargs: Any) -> None:
+        '''Deletes a chunk or a list of chunks from the data store.
+
+        Args:
+            query (dict[str, Any]): The query to filter the chunks to delete.
+                For example, `{"term": {"metadata.id": "doc123"}}`.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            None
+        '''

gllm_datastore/vector_data_store/vector_data_store.pyi

@@ -0,0 +1,73 @@
+import abc
+from abc import ABC, abstractmethod
+from gllm_core.schema.chunk import Chunk as Chunk
+from gllm_datastore.constants import DEFAULT_TOP_K as DEFAULT_TOP_K
+from typing import Any
+
+class BaseVectorDataStore(ABC, metaclass=abc.ABCMeta):
+    """Abstract base class for vector data stores in the retrieval system.
+
+    This class defines the interface for all vector data store implementations.
+    Subclasses must implement the `query` and `query_by_id` methods.
+    """
+    @abstractmethod
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Executes a query on the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            query (str): The query string to execute.
+            top_k (int, optional): The maximum number of results to return. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional parameters for the query.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of query results.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Retrieves chunks by their IDs.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            id_ (str | list[str]): A single ID or a list of IDs to retrieve.
+
+        Returns:
+            list[Chunk]: A list of retrieved chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def add_chunks(self, chunk: Chunk | list[Chunk], **kwargs) -> list[str]:
+        """Adds a chunk or a list of chunks in the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            chunk (Chunk | list[Chunk]): A single chunk or a list of chunks to index.
+            **kwargs: Additional keyword arguments to pass to the method.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_chunks(self, **kwargs: Any) -> None:
+        """Deletes a chunk or a list of chunks from the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            kwargs: Additional keyword arguments to pass to the method.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """

gllm_datastore.build/.gitignore

@@ -0,0 +1 @@
+*

gllm_datastore.pyi

@@ -0,0 +1,63 @@
+# This file was generated by Nuitka
+
+# Stubs included by default
+
+
+__name__ = ...
+
+
+
+# Modules used internally, to allow implicit dependencies to be seen:
+import os
+import asyncio
+import functools
+import abc
+import enum
+import typing
+import gllm_core
+import gllm_core.utils
+import gllm_core.utils.imports
+import gllm_datastore.utils.convert_ttl_to_seconds
+import Levenshtein
+import warnings
+import gzip
+import json
+import pickle
+import shutil
+import time
+import gllm_core.utils.logger_manager
+import ntpath
+import datetime
+import redis
+import hashlib
+import llama_index
+import llama_index.core
+import llama_index.core.graph_stores
+import llama_index.core.graph_stores.types
+import llama_index.graph_stores
+import llama_index.graph_stores.neo4j
+import nebula3
+import nebula3.Config
+import nebula3.gclient
+import nebula3.gclient.net
+import neo4j
+import sqlalchemy
+import sqlalchemy.engine
+import pandas
+import sqlalchemy.exc
+import sqlalchemy.orm
+import pydantic
+import uuid
+import gllm_core.schema
+import langchain_core
+import langchain_core.documents
+import sys
+import gllm_core.schema.chunk
+import langchain_core.embeddings
+import chromadb
+import chromadb.types
+import langchain_chroma
+import pysqlite3
+import elasticsearch
+import langchain_elasticsearch
+import langchain_elasticsearch.vectorstores

gllm_datastore_binary-0.0.15.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.1
+Name: gllm-datastore-binary
+Version: 0.0.15
+Summary: A library containing data store components for Gen AI applications.
+Author: Berty C L Tobing
+Author-email: berty.c.l.tobing@gdplabs.id
+Requires-Python: >=3.11,<4.0
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: chroma
+Provides-Extra: elasticsearch
+Provides-Extra: fuzzy
+Provides-Extra: kg
+Provides-Extra: redis
+Requires-Dist: Jinja2 (>=3.1.4,<4.0.0) ; extra == "kg"
+Requires-Dist: chromadb (>=0.6.3,<0.7.0) ; extra == "chroma"
+Requires-Dist: gllm-core-binary
+Requires-Dist: langchain-chroma (>=0.2.2,<0.3.0) ; extra == "chroma"
+Requires-Dist: langchain-elasticsearch (==0.3.0) ; extra == "elasticsearch"
+Requires-Dist: llama-index-core (>=0.12.0,<0.13.0) ; extra == "kg"
+Requires-Dist: llama-index-graph-stores-nebula (>=0.4.0,<0.5.0) ; extra == "kg"
+Requires-Dist: llama-index-graph-stores-neo4j (>=0.4.0,<0.5.0) ; extra == "kg"
+Requires-Dist: nebula3-python (>=3.8.3,<4.0.0)
+Requires-Dist: neo4j (>=5.28.1,<6.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pysqlite3-binary (>=0.5.4,<0.6.0) ; sys_platform == "linux"
+Requires-Dist: python-levenshtein (==0.26.1) ; extra == "fuzzy"
+Requires-Dist: redis (==5.2.1) ; extra == "redis"
+Requires-Dist: sqlalchemy (>=2.0.37,<3.0.0)
+Description-Content-Type: text/markdown
+
+# GLLM Datastore
+
+## Description
+
+A library for managing data stores for Gen AI applications.
+
+## Installation
+
+1. Python v3.11 or above:
+
+You can install Python using [Miniconda](https://docs.anaconda.com/free/miniconda/index.html).
+
+2. Make sure you're in the `base` conda environment:
+```bash
+conda activate
+```
+
+3. [Poetry](https://python-poetry.org/docs/) v1.8.1 or above:
+
+You can install Poetry using cURL (you need Python to install Poetry):
+```bash
+curl -sSL https://install.python-poetry.org | python3 -
+```
+
+4. Install the library using Poetry:
+```bash
+# Latest
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git#subdirectory=libs/gllm-datastore"
+
+# Specific version
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git@gllm_datastore-v0.0.1-beta.1#subdirectory=libs/gllm-datastore"
+
+# Specific Branch Name
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git@<BRANCH NAME>#subdirectory=libs/gllm-datastore"
+
+# With extra dependencies
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git#subdirectory=libs/gllm-datastore" --extras "extra1 extra2"
+```
+
+5. At this step, you can deactivate Miniconda environment as Poetry will create and manage its own virtual environment for you.
+```bash
+conda deactivate
+```
+
+## Managing Dependencies
+1. Go to root folder of `gllm-datastore` module, e.g. `cd libs/gllm-datastore`.
+2. Run `poetry shell` to create a virtual environment.
+3. Run `poetry lock` to create a lock file if you haven't done it yet.
+4. Run `poetry install` to install the `gllm-datastore` requirements for the first time.
+5. Run `poetry update` if you update any dependency module version at `pyproject.toml`.
+
+
+## Contributing
+Please refer to this [Python Style Guide](https://docs.google.com/document/d/1uRggCrHnVfDPBnG641FyQBwUwLoFw0kTzNqRm92vUwM/edit?usp=sharing)
+to get information about code style, documentation standard, and SCA that you need to use when contributing to this project
+
+1. Activate `pre-commit` hooks using `pre-commit install`
+2. Run `poetry shell` to create a virtual environment.
+3. Run `poetry lock` to create a lock file if you haven't done it yet.
+4. Run `poetry install` to install the `gllm-datastore` requirements for the first time.
+5. Run `which python` to get the path to be referenced at Visual Studio Code interpreter path (`Ctrl`+`Shift`+`P` or `Cmd`+`Shift`+`P`)
+6. Try running the unit test to see if it's working:
+```bash
+poetry run pytest -s tests/unit_tests/
+```
+

gllm_datastore_binary-0.0.15.dist-info/RECORD

@@ -0,0 +1,38 @@
+gllm_datastore/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/cache_data_store/__init__.pyi,sha256=pT0UrQbw6z9xHO3EwvM4wkug33jWLaxxj7tbEz6cQx0,450
+gllm_datastore/cache_data_store/cache_data_store.pyi,sha256=A5MZekHV4i03d0KIJHUWB3kFe040ycRqBpaQdJMD0P4,6505
+gllm_datastore/cache_data_store/cache_data_store_utils.pyi,sha256=5ndFYWZ4yG8oyxmdq08SqVr7lejID7IXIvZTRDS3hsI,53
+gllm_datastore/cache_data_store/file_system_cache_data_store.pyi,sha256=QHOSCRVQUQTqixXedN3gGsjZ0fR4zkVmHLdN6siHVjk,3483
+gllm_datastore/cache_data_store/in_memory_cache_data_store.pyi,sha256=u81kwqFj5t3U7jwJlMbLLASs4sIXaC_JkPKkNBiSPAU,2149
+gllm_datastore/cache_data_store/redis_cache_data_store.pyi,sha256=betNBcLf2WZySneuGLIpnN2G4mL3UKogUFpBSz6ohqs,2438
+gllm_datastore/cache_data_store/utils.pyi,sha256=PlgCLEsmQV_lUkkn0XnaQilrOxzsyM6qG4N-HOZWnHY,1377
+gllm_datastore/constants.pyi,sha256=s0K1tUBRBFDfjODvZwFFUPxWdtAZNwyciINmIjKU-0E,50
+gllm_datastore/graph_data_store/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/graph_data_store/graph_data_store.pyi,sha256=EUrp48f1gYct3QuWMD6qSdKhFJZXikSHMVhoOWUFmoI,3398
+gllm_datastore/graph_data_store/graph_rag_data_store.pyi,sha256=eYuoSU46xm-0I3EC3m_hmDNqX9D8EXpXDiex-Mlox_E,917
+gllm_datastore/graph_data_store/llama_index_graph_rag_data_store.pyi,sha256=1FM90HKSzD7UdYROOrcnRZICmhz3Qt8SrNo4xPG4hwA,721
+gllm_datastore/graph_data_store/llama_index_neo4j_graph_rag_data_store.pyi,sha256=giAhUk2hHhKQm8mVdogne0QCf18zadnn3wr-BKpelPk,1120
+gllm_datastore/graph_data_store/nebula_graph_data_store.pyi,sha256=jYtal--cvV_aT9Zw3xpT3hd80ZAj4oOJfbd0ugEb4pw,4975
+gllm_datastore/graph_data_store/neo4j_graph_data_store.pyi,sha256=UXNH_HAqVCjvF_P7kv8XqFm4i8LR1OnhLrO_9aHjJLY,3545
+gllm_datastore/sql_data_store/__init__.pyi,sha256=EHEc8NNcXMn2zAECQzCOW7ZnFmlg50j4e05hfIvAC24,426
+gllm_datastore/sql_data_store/adapter/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/sql_data_store/adapter/sqlalchemy_adapter.pyi,sha256=9u670RQQYCMpvPDf-cNyRrDhQheWIDhR-kr6fFRCXu4,1417
+gllm_datastore/sql_data_store/constants.pyi,sha256=m38bKAzPCFog92Mv_pkM-6S5A83_AXWARTPfXXytPBw,139
+gllm_datastore/sql_data_store/sql_data_store.pyi,sha256=PAYBQW6JniZ0h2WBmWL2phqc1FsWUuHn2GyJjxZ_gKo,3839
+gllm_datastore/sql_data_store/sqlalchemy_data_store.pyi,sha256=Ajk19DVerVT_ZTChHYIH-J95nuIcBRQhuFvvAaedsCE,398
+gllm_datastore/sql_data_store/sqlalchemy_sql_data_store.pyi,sha256=RYYJt93xofbckBvcUp5HhosEjEg3KIFFtHUc78beTac,8288
+gllm_datastore/sql_data_store/types.pyi,sha256=DoaI5LCOMVP2zxHpD2yimFovhWFP_dBe3K5lv_InxhI,1061
+gllm_datastore/utils/__init__.pyi,sha256=YE0R6kzfJGi0LVj12JDTYJBTXr2sFdDnPUKw1k6V_L4,222
+gllm_datastore/utils/converter.pyi,sha256=H4_O-diiWw33TDEqHWB24SR25tpbuSE6Lu6-G7o_X28,610
+gllm_datastore/utils/ttl.pyi,sha256=CPGdxD3HpWABw149t-kS_usPReNMj-jRaVRnrd8TDWk,778
+gllm_datastore/vector_data_store/__init__.pyi,sha256=U1x8eo3OJ5jTjoC4fz2zgyapxZmn-e3ZtI8D32PQ_-g,331
+gllm_datastore/vector_data_store/chroma_vector_data_store.pyi,sha256=NQNTcOztwiHhzApsweT_5xotEk_CUTZHYuP7kl8WEjA,6529
+gllm_datastore/vector_data_store/elasticsearch_data_store.pyi,sha256=1-lXVSacRhm0FIwykJwUI9lOwRQmrYCVF5cYOx7qjaw,450
+gllm_datastore/vector_data_store/elasticsearch_vector_data_store.pyi,sha256=Bja4Woio3cAw5SY1FPQRRYAjbPzenb02iYDxePw6i7o,8086
+gllm_datastore/vector_data_store/vector_data_store.pyi,sha256=GoeW_BxpZywJ11oEWHEXw7P0kap7LWxE-3p3KpaHhk4,2695
+gllm_datastore.build/.gitignore,sha256=aEiIwOuxfzdCmLZe4oB1JsBmCUxwG8x-u-HBCV9JT8E,1
+gllm_datastore.cp311-win_amd64.pyd,sha256=dfuKJzExRDfGrWYP2MVQT7H2Nb4iQv_khRygh9UHS7A,1121792
+gllm_datastore.pyi,sha256=gSGm1EVJP80w3dv9DffuYdvDilHCT67qkJc8dPtJpFA,1299
+gllm_datastore_binary-0.0.15.dist-info/METADATA,sha256=7rSZVe65XYS1QNvliXvw0gnZDHpcrIPtnZACyeMIxis,3900
+gllm_datastore_binary-0.0.15.dist-info/WHEEL,sha256=-FZBVKyKauScY3vLa8vJR6hBCpAJfFykw2MOwlNKr1g,98
+gllm_datastore_binary-0.0.15.dist-info/RECORD,,

gllm_datastore_binary-0.0.15.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: false
+Tag: cp311-cp311-win_amd64