gllm-datastore-binary 0.0.15__cp312-cp312-macosx_14_0_arm64.whl

gllm_datastore/vector_data_store/elasticsearch_vector_data_store.pyi

@@ -0,0 +1,140 @@
+from _typeshed import Incomplete
+from gllm_core.schema import Chunk
+from gllm_datastore.constants import DEFAULT_REQUEST_TIMEOUT as DEFAULT_REQUEST_TIMEOUT, DEFAULT_TOP_K as DEFAULT_TOP_K
+from gllm_datastore.utils.converter import from_langchain as from_langchain, to_langchain as to_langchain
+from gllm_datastore.vector_data_store.vector_data_store import BaseVectorDataStore as BaseVectorDataStore
+from langchain_core.embeddings import Embeddings as Embeddings
+from typing import Any
+
+class ElasticsearchVectorDataStore(BaseVectorDataStore):
+    """DataStore for interacting with Elasticsearch.
+
+    This class provides methods for executing queries and retrieving documents
+    from Elasticsearch. It relies on the LangChain's ElasticsearchStore  for
+    vector operations and the underlying Elasticsearch client management.
+
+    Attributes:
+        store (ElasticsearchStore): The ElasticsearchStore instance for vector operations.
+        index_name (str): The name of the Elasticsearch index.
+        logger (Logger): The logger object.
+    """
+    index_name: Incomplete
+    store: Incomplete
+    logger: Incomplete
+    def __init__(self, index_name: str, embedding: Embeddings | None = None, connection: Any | None = None, url: str | None = None, cloud_id: str | None = None, user: str | None = None, api_key: str | None = None, password: str | None = None, vector_query_field: str = 'vector', query_field: str = 'text', distance_strategy: str | None = None, strategy: Any | None = None, request_timeout: int = ...) -> None:
+        '''Initializes an instance of the ElasticsearchVectorDataStore class.
+
+        Args:
+            index_name (str): The name of the Elasticsearch index.
+            embedding (Embeddings | None, optional): The Embeddings object for vector operations. Defaults to None.
+            connection (Any | None, optional): The Elasticsearch connection object. Defaults to None.
+            url (str | None, optional): The URL of the Elasticsearch server. Defaults to None.
+            cloud_id (str | None, optional): The cloud ID of the Elasticsearch cluster. Defaults to None.
+            user (str | None, optional): The username for authentication. Defaults to None.
+            api_key (str | None, optional): The API key for authentication. Defaults to None.
+            password (str | None, optional): The password for authentication. Defaults to None.
+            vector_query_field (str, optional): The field name for vector queries. Defaults to "vector".
+            query_field (str, optional): The field name for text queries. Defaults to "text".
+            distance_strategy (str | None, optional): The distance strategy for retrieval. Defaults to None.
+            strategy (Any | None, optional): The retrieval strategy for retrieval. Defaults to None, in which case
+                DenseVectorStrategy() is used.
+            request_timeout (int, optional): The request timeout. Defaults to DEFAULT_REQUEST_TIMEOUT.
+        '''
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Queries the Elasticsearch data store.
+
+        Args:
+            query (str): The query string.
+            top_k (int, optional): The number of top results to retrieve. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional retrieval parameters. Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of Chunk objects representing the retrieved documents.
+        """
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Queries the data store by ID and returns a list of Chunk objects.
+
+        Args:
+            id_: The ID of the document to query.
+
+        Returns:
+            A list of Chunk objects representing the queried documents.
+
+        Note:
+            This method not implement yet. Because the ElasticsearchStore
+            still not implement the get_by_ids method yet.
+        """
+    async def autocomplete(self, query: str, field: str, size: int = 20, fuzzy_tolerance: int = 1, min_prefix_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Provides suggestions based on a prefix query for a specific field.
+
+        Args:
+            query (str): The query string.
+            field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            fuzzy_tolerance (int, optional): The level of fuzziness for suggestions. Defaults to 1.
+            min_prefix_length (int, optional): The minimum prefix length to trigger fuzzy matching. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def autosuggest(self, query: str, search_fields: list[str], autocomplete_field: str, size: int = 20, min_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Generates suggestions across multiple fields using a multi_match query to broaden the search criteria.
+
+        Args:
+            query (str): The query string.
+            search_fields (list[str]): The fields to search for.
+            autocomplete_field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            min_length (int, optional): The minimum length of the query. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def shingles(self, query: str, field: str, size: int = 20, min_length: int = 3, filter_query: dict[str, Any] | None = None) -> list[str]:
+        """Searches using shingles for prefix and fuzzy matching.
+
+        Args:
+            query (str): The query string.
+            field (str): The field name for autocomplete.
+            size (int, optional): The number of suggestions to retrieve. Defaults to 20.
+            min_length (int, optional): The minimum length of the query. Defaults to 3.
+            filter_query (dict[str, Any] | None, optional): The filter query. Defaults to None.
+
+        Returns:
+            list[str]: A list of suggestions.
+        """
+    async def add_chunks(self, chunk: Chunk | list[Chunk], **kwargs: Any) -> list[str]:
+        """Adds a chunk or a list of chunks to the data store.
+
+        Args:
+            chunk (Chunk | list[Chunk]): The chunk or list of chunks to add.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added chunks.
+        """
+    async def add_embeddings(self, text_embeddings: list[tuple[str, list[float]]], metadatas: list[dict] | None = None, ids: list[str] | None = None, **kwargs) -> list[str]:
+        """Adds text embeddings to the data store.
+
+        Args:
+            text_embeddings (list[tuple[str, list[float]]]): Pairs of string and embedding to add to the store.
+            metadatas (list[dict], optional): Optional list of metadatas associated with the texts. Defaults to None.
+            ids (list[str], optional): Optional list of unique IDs. Defaults to None.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added embeddings.
+        """
+    async def delete_chunks(self, query: dict[str, Any], **kwargs: Any) -> None:
+        '''Deletes a chunk or a list of chunks from the data store.
+
+        Args:
+            query (dict[str, Any]): The query to filter the chunks to delete.
+                For example, `{"term": {"metadata.id": "doc123"}}`.
+            kwargs (Any): Additional keyword arguments.
+
+        Returns:
+            None
+        '''

gllm_datastore/vector_data_store/vector_data_store.pyi

@@ -0,0 +1,73 @@
+import abc
+from abc import ABC, abstractmethod
+from gllm_core.schema.chunk import Chunk as Chunk
+from gllm_datastore.constants import DEFAULT_TOP_K as DEFAULT_TOP_K
+from typing import Any
+
+class BaseVectorDataStore(ABC, metaclass=abc.ABCMeta):
+    """Abstract base class for vector data stores in the retrieval system.
+
+    This class defines the interface for all vector data store implementations.
+    Subclasses must implement the `query` and `query_by_id` methods.
+    """
+    @abstractmethod
+    async def query(self, query: str, top_k: int = ..., retrieval_params: dict[str, Any] | None = None) -> list[Chunk]:
+        """Executes a query on the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            query (str): The query string to execute.
+            top_k (int, optional): The maximum number of results to return. Defaults to DEFAULT_TOP_K.
+            retrieval_params (dict[str, Any] | None, optional): Additional parameters for the query.
+                Defaults to None.
+
+        Returns:
+            list[Chunk]: A list of query results.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def query_by_id(self, id_: str | list[str]) -> list[Chunk]:
+        """Retrieves chunks by their IDs.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            id_ (str | list[str]): A single ID or a list of IDs to retrieve.
+
+        Returns:
+            list[Chunk]: A list of retrieved chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def add_chunks(self, chunk: Chunk | list[Chunk], **kwargs) -> list[str]:
+        """Adds a chunk or a list of chunks in the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            chunk (Chunk | list[Chunk]): A single chunk or a list of chunks to index.
+            **kwargs: Additional keyword arguments to pass to the method.
+
+        Returns:
+            list[str]: A list of unique identifiers (IDs) assigned to the added chunks.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """
+    @abstractmethod
+    async def delete_chunks(self, **kwargs: Any) -> None:
+        """Deletes a chunk or a list of chunks from the data store.
+
+        This method must be implemented by subclasses.
+
+        Args:
+            kwargs: Additional keyword arguments to pass to the method.
+
+        Raises:
+            NotImplementedError: If the method is not implemented.
+        """

gllm_datastore.build/.gitignore

@@ -0,0 +1 @@
+*

gllm_datastore.pyi

@@ -0,0 +1,63 @@
+# This file was generated by Nuitka
+
+# Stubs included by default
+
+
+__name__ = ...
+
+
+
+# Modules used internally, to allow implicit dependencies to be seen:
+import os
+import asyncio
+import functools
+import abc
+import enum
+import typing
+import gllm_core
+import gllm_core.utils
+import gllm_core.utils.imports
+import gllm_datastore.utils.convert_ttl_to_seconds
+import Levenshtein
+import warnings
+import gzip
+import json
+import pickle
+import shutil
+import time
+import gllm_core.utils.logger_manager
+import posixpath
+import datetime
+import redis
+import hashlib
+import llama_index
+import llama_index.core
+import llama_index.core.graph_stores
+import llama_index.core.graph_stores.types
+import llama_index.graph_stores
+import llama_index.graph_stores.neo4j
+import nebula3
+import nebula3.Config
+import nebula3.gclient
+import nebula3.gclient.net
+import neo4j
+import sqlalchemy
+import sqlalchemy.engine
+import pandas
+import sqlalchemy.exc
+import sqlalchemy.orm
+import pydantic
+import uuid
+import gllm_core.schema
+import langchain_core
+import langchain_core.documents
+import sys
+import gllm_core.schema.chunk
+import langchain_core.embeddings
+import chromadb
+import chromadb.types
+import langchain_chroma
+import pysqlite3
+import elasticsearch
+import langchain_elasticsearch
+import langchain_elasticsearch.vectorstores

gllm_datastore_binary-0.0.15.dist-info/METADATA

@@ -0,0 +1,98 @@
+Metadata-Version: 2.1
+Name: gllm-datastore-binary
+Version: 0.0.15
+Summary: A library containing data store components for Gen AI applications.
+Author: Berty C L Tobing
+Author-email: berty.c.l.tobing@gdplabs.id
+Requires-Python: >=3.11,<3.13
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Provides-Extra: chroma
+Provides-Extra: elasticsearch
+Provides-Extra: fuzzy
+Provides-Extra: kg
+Provides-Extra: redis
+Requires-Dist: Jinja2 (>=3.1.4,<4.0.0) ; extra == "kg"
+Requires-Dist: chromadb (>=0.6.3,<0.7.0) ; extra == "chroma"
+Requires-Dist: gllm-core-binary
+Requires-Dist: langchain-chroma (>=0.2.2,<0.3.0) ; extra == "chroma"
+Requires-Dist: langchain-elasticsearch (==0.3.0) ; extra == "elasticsearch"
+Requires-Dist: llama-index-core (>=0.12.0,<0.13.0) ; extra == "kg"
+Requires-Dist: llama-index-graph-stores-nebula (>=0.4.0,<0.5.0) ; extra == "kg"
+Requires-Dist: llama-index-graph-stores-neo4j (>=0.4.0,<0.5.0) ; extra == "kg"
+Requires-Dist: nebula3-python (>=3.8.3,<4.0.0)
+Requires-Dist: neo4j (>=5.28.1,<6.0.0)
+Requires-Dist: pandas (==2.2.2)
+Requires-Dist: pysqlite3-binary (>=0.5.4,<0.6.0) ; sys_platform == "linux"
+Requires-Dist: python-levenshtein (==0.26.1) ; extra == "fuzzy"
+Requires-Dist: redis (==5.2.1) ; extra == "redis"
+Requires-Dist: sqlalchemy (>=2.0.37,<3.0.0)
+Description-Content-Type: text/markdown
+
+# GLLM Datastore
+
+## Description
+
+A library for managing data stores for Gen AI applications.
+
+## Installation
+
+1. Python v3.11 or above:
+
+You can install Python using [Miniconda](https://docs.anaconda.com/free/miniconda/index.html).
+
+2. Make sure you're in the `base` conda environment:
+```bash
+conda activate
+```
+
+3. [Poetry](https://python-poetry.org/docs/) v1.8.1 or above:
+
+You can install Poetry using cURL (you need Python to install Poetry):
+```bash
+curl -sSL https://install.python-poetry.org | python3 -
+```
+
+4. Install the library using Poetry:
+```bash
+# Latest
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git#subdirectory=libs/gllm-datastore"
+
+# Specific version
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git@gllm_datastore-v0.0.1-beta.1#subdirectory=libs/gllm-datastore"
+
+# Specific Branch Name
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git@<BRANCH NAME>#subdirectory=libs/gllm-datastore"
+
+# With extra dependencies
+poetry add "git+ssh://git@github.com/GDP-ADMIN/gen-ai-internal.git#subdirectory=libs/gllm-datastore" --extras "extra1 extra2"
+```
+
+5. At this step, you can deactivate Miniconda environment as Poetry will create and manage its own virtual environment for you.
+```bash
+conda deactivate
+```
+
+## Managing Dependencies
+1. Go to root folder of `gllm-datastore` module, e.g. `cd libs/gllm-datastore`.
+2. Run `poetry shell` to create a virtual environment.
+3. Run `poetry lock` to create a lock file if you haven't done it yet.
+4. Run `poetry install` to install the `gllm-datastore` requirements for the first time.
+5. Run `poetry update` if you update any dependency module version at `pyproject.toml`.
+
+
+## Contributing
+Please refer to this [Python Style Guide](https://docs.google.com/document/d/1uRggCrHnVfDPBnG641FyQBwUwLoFw0kTzNqRm92vUwM/edit?usp=sharing)
+to get information about code style, documentation standard, and SCA that you need to use when contributing to this project
+
+1. Activate `pre-commit` hooks using `pre-commit install`
+2. Run `poetry shell` to create a virtual environment.
+3. Run `poetry lock` to create a lock file if you haven't done it yet.
+4. Run `poetry install` to install the `gllm-datastore` requirements for the first time.
+5. Run `which python` to get the path to be referenced at Visual Studio Code interpreter path (`Ctrl`+`Shift`+`P` or `Cmd`+`Shift`+`P`)
+6. Try running the unit test to see if it's working:
+```bash
+poetry run pytest -s tests/unit_tests/
+```
+

gllm_datastore_binary-0.0.15.dist-info/RECORD

@@ -0,0 +1,38 @@
+gllm_datastore/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/cache_data_store/__init__.pyi,sha256=U_47XPcWYzeslxlIq9FxIZqDh0-HRGsGF4KilrBVnM0,445
+gllm_datastore/cache_data_store/cache_data_store.pyi,sha256=N3MrkqBX5vtX55uqrnYU1ST7CGblozarXGBe8cIY9KY,6359
+gllm_datastore/cache_data_store/cache_data_store_utils.pyi,sha256=ave97lxWbwEy6LdZi3t_e25vuj9ee0ifCzUEnRKxcek,52
+gllm_datastore/cache_data_store/file_system_cache_data_store.pyi,sha256=PSPg1UY6UAeiJLZdx-AMR9NyJPOtbow2tJW06f7q1_Y,3421
+gllm_datastore/cache_data_store/in_memory_cache_data_store.pyi,sha256=X0DbarCRvCYKAgWklBU-rJdrWiUbrpn8N8IZ6tp7uy0,2106
+gllm_datastore/cache_data_store/redis_cache_data_store.pyi,sha256=o1tUjQuJmmSgiZP9jKOIJKKi9K30yhHEhIuCgj62JQE,2390
+gllm_datastore/cache_data_store/utils.pyi,sha256=-58dSDdWbd2DIlcrer8B0mj-DG1LTiWFdi1m0NQp_GU,1341
+gllm_datastore/constants.pyi,sha256=phT3qZc6RNnIw6m5MwXuJMPMRZDc_74wWvSCnSKe8jA,48
+gllm_datastore/graph_data_store/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/graph_data_store/graph_data_store.pyi,sha256=WCfwuVYpOPr1cVN1YbnjY2HQVo9IiKMShVimkbz90pY,3318
+gllm_datastore/graph_data_store/graph_rag_data_store.pyi,sha256=p6KOSnPJdoM7EhGXlUm7FAwXdtjRfpVH0Wwnr0m1JqU,889
+gllm_datastore/graph_data_store/llama_index_graph_rag_data_store.pyi,sha256=doHkYIQ85WW-P787PmzbP7Bp_4IQy_jVn63cck_93vw,703
+gllm_datastore/graph_data_store/llama_index_neo4j_graph_rag_data_store.pyi,sha256=ErjX8WabRS7b0kk5QEEU2eJLl5LSycJXtcFaD-covG0,1094
+gllm_datastore/graph_data_store/nebula_graph_data_store.pyi,sha256=eBW4Hk7Jhz3GmLItkZAdKnPt6VtOBOiBAwVrZdVAk-c,4863
+gllm_datastore/graph_data_store/neo4j_graph_data_store.pyi,sha256=qn4FJRenxI0Ih1vPmuszGkOORolFpIm6o2iIBDBB8bI,3464
+gllm_datastore/sql_data_store/__init__.pyi,sha256=WNg3UnqwdtYC9EJKxU1aYvg34LZSRk5hP8HpzSH47qw,421
+gllm_datastore/sql_data_store/adapter/__init__.pyi,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
+gllm_datastore/sql_data_store/adapter/sqlalchemy_adapter.pyi,sha256=DYCOAlPpPVdBN0Pr6b5xyq9vtzEGcsiOIOQr9WbfMsY,1387
+gllm_datastore/sql_data_store/constants.pyi,sha256=9iF_A9NPu28nz-i19ltaMi-Eq4iZF7V8nMjab2ajugw,133
+gllm_datastore/sql_data_store/sql_data_store.pyi,sha256=R6TJ4W2OaFCThW247GN1mi1V9YqGLGZ1mAU9_yL4rGI,3752
+gllm_datastore/sql_data_store/sqlalchemy_data_store.pyi,sha256=7Es7V3hoIPFPUNgVhIXJn04EIdBgmcMhUFlOhfHvvxM,389
+gllm_datastore/sql_data_store/sqlalchemy_sql_data_store.pyi,sha256=zgW33fJaIxDwodNNZYLh18hAR6eh1bzJJu5Ih2od9U8,8105
+gllm_datastore/sql_data_store/types.pyi,sha256=AWFwt-_puEXMS8s8EPeKm20J0J2YGuQNuZ3cGI_0oaU,1031
+gllm_datastore/utils/__init__.pyi,sha256=bH7ZwKoCX3I5IoPLPW0EJ0D5WDWHRBpGsxk5AMy-s0o,218
+gllm_datastore/utils/converter.pyi,sha256=omWN28Rh_Q7u3g46NkEcbXYagZc5-br8MBhT-heYxHs,589
+gllm_datastore/utils/ttl.pyi,sha256=zUvLTHhvgRtyKRdjdJk918qYiZkDwWQrbROl00TbpvQ,753
+gllm_datastore/vector_data_store/__init__.pyi,sha256=EAaorTfSpjIPqunbgSe_tVllahAvJJfjgfMe8fN--Tc,327
+gllm_datastore/vector_data_store/chroma_vector_data_store.pyi,sha256=QZDdP5qzxhRIY5ewRPCz4g0PWOMFY0i_CzqDgKXJCjw,6410
+gllm_datastore/vector_data_store/elasticsearch_data_store.pyi,sha256=gNLFrArc69CbhuAbGaEJE9wPqIWUVn0DJwwT8_M1zKc,441
+gllm_datastore/vector_data_store/elasticsearch_vector_data_store.pyi,sha256=fSvBWp3YXJWlKY52saFEKm3RcPDxIgFT6NHlQDe7T9s,7946
+gllm_datastore/vector_data_store/vector_data_store.pyi,sha256=f7LXNzFxpYR-wdct2Q0DPDvfkXYtTGerqsoreINyAG0,2622
+gllm_datastore.build/.gitignore,sha256=aEiIwOuxfzdCmLZe4oB1JsBmCUxwG8x-u-HBCV9JT8E,1
+gllm_datastore.cpython-312-darwin.so,sha256=4jXgv_D6AtMSY92hpFZWSnb3dQUWK7Fubfr65NwGLNw,1387576
+gllm_datastore.pyi,sha256=FUSlAyC9KQsOuVVDxozFw-FWvd478MmaeFwdItKFiQU,1302
+gllm_datastore_binary-0.0.15.dist-info/METADATA,sha256=c1zlDaqZBl9PMHlrT-5oBAtl_XfHPZ2uwx3s4yxFN0I,3901
+gllm_datastore_binary-0.0.15.dist-info/WHEEL,sha256=yhFWDruQeL16y7AZmR-YzCRfHXG6441RPQ3U5xtqaLo,106
+gllm_datastore_binary-0.0.15.dist-info/RECORD,,

gllm_datastore_binary-0.0.15.dist-info/WHEEL

@@ -0,0 +1,4 @@
+Wheel-Version: 1.0
+Generator: poetry-core 1.9.0
+Root-Is-Purelib: false
+Tag: cp312-cp312-macosx_14_0_arm64