llama-index-vector-stores-bigquery 0.1.0__tar.gz

llama_index_vector_stores_bigquery-0.1.0/.gitignore

@@ -0,0 +1,153 @@
+llama_index/_static
+.DS_Store
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+bin/
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+etc/
+include/
+lib/
+lib64/
+parts/
+sdist/
+share/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+.ruff_cache
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+notebooks/
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+pyvenv.cfg
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# Jetbrains
+.idea
+modules/
+*.swp
+
+# VsCode
+.vscode
+
+# pipenv
+Pipfile
+Pipfile.lock
+
+# pyright
+pyrightconfig.json

llama_index_vector_stores_bigquery-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) Jerry Liu
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

llama_index_vector_stores_bigquery-0.1.0/PKG-INFO

@@ -0,0 +1,44 @@
+Metadata-Version: 2.4
+Name: llama-index-vector-stores-bigquery
+Version: 0.1.0
+Summary: llama-index vector_stores bigquery vector search integration
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: <4.0,>=3.9
+Requires-Dist: google-cloud-bigquery<4,>=3.34.0
+Requires-Dist: llama-index-core<0.14,>=0.13.0
+Description-Content-Type: text/markdown
+
+# LlamaIndex Vector_Stores Integration: BigQuery
+
+Vector store index using Google BigQuery.
+Supports efficient storage and querying of vector embeddings using BigQuery's native vector search capabilities.
+For more information, see the official [BigQuery Vector Search Documentation](https://cloud.google.com/bigquery/docs/vector-search-intro)
+
+## 🔐 Required IAM Permissions
+
+To use this integration, ensure your account has the following permissions:
+
+- `roles/bigquery.dataOwner` (BigQuery Data Owner)
+- `roles/bigquery.dataEditor` (BigQuery Data Editor)
+
+## 🔧 Installation
+
+```bash
+pip install llama-index-vector-stores-bigquery
+```
+
+## 💻 Example Usage
+
+```python
+from google.cloud.bigquery import Client
+from llama_index.vector_stores.bigquery import BigQueryVectorStore
+
+client = Client()
+
+vector_store = BigQueryVectorStore(
+    table_id="my_bigquery_table",
+    dataset_id="my_bigquery_dataset",
+    bigquery_client=client,
+)
+```

llama_index_vector_stores_bigquery-0.1.0/README.md

@@ -0,0 +1,33 @@
+# LlamaIndex Vector_Stores Integration: BigQuery
+
+Vector store index using Google BigQuery.
+Supports efficient storage and querying of vector embeddings using BigQuery's native vector search capabilities.
+For more information, see the official [BigQuery Vector Search Documentation](https://cloud.google.com/bigquery/docs/vector-search-intro)
+
+## 🔐 Required IAM Permissions
+
+To use this integration, ensure your account has the following permissions:
+
+- `roles/bigquery.dataOwner` (BigQuery Data Owner)
+- `roles/bigquery.dataEditor` (BigQuery Data Editor)
+
+## 🔧 Installation
+
+```bash
+pip install llama-index-vector-stores-bigquery
+```
+
+## 💻 Example Usage
+
+```python
+from google.cloud.bigquery import Client
+from llama_index.vector_stores.bigquery import BigQueryVectorStore
+
+client = Client()
+
+vector_store = BigQueryVectorStore(
+    table_id="my_bigquery_table",
+    dataset_id="my_bigquery_dataset",
+    bigquery_client=client,
+)
+```

llama_index_vector_stores_bigquery-0.1.0/llama_index/vector_stores/bigquery/__init__.py

@@ -0,0 +1,3 @@
+from llama_index.vector_stores.bigquery.base import BigQueryVectorStore
+
+__all__ = ["BigQueryVectorStore"]

llama_index_vector_stores_bigquery-0.1.0/llama_index/vector_stores/bigquery/base.py

@@ -0,0 +1,547 @@
+"""
+Google BigQuery Vector Search.
+
+BigQuery Vector Search is a fully managed feature of BigQuery that enables fast,
+scalable similarity search over high-dimensional embeddings using approximate
+nearest neighbor methods. For more information visit:
+https://cloud.google.com/bigquery/docs/vector-search-intro
+"""
+
+import logging
+from enum import Enum
+from typing import Any, Dict, List, Optional, Union
+
+from google.auth import credentials
+from google.cloud import bigquery
+from llama_index.core.bridge.pydantic import PrivateAttr
+from llama_index.core.schema import BaseNode, MetadataMode, TextNode
+from llama_index.core.vector_stores.types import (
+    BasePydanticVectorStore,
+    MetadataFilters,
+    VectorStoreQuery,
+    VectorStoreQueryResult,
+)
+from llama_index.core.vector_stores.utils import (
+    metadata_dict_to_node,
+    node_to_metadata_dict,
+)
+from pydantic import BaseModel
+
+from llama_index.vector_stores.bigquery.utils import build_where_clause_and_params
+
+
+_logger = logging.getLogger(__name__)
+
+
+class _BigQueryRow(BaseModel):
+    node_id: str
+    embedding: List[float]
+    text: str
+    metadata: Dict[str, Any]
+    distance: Optional[float] = None
+
+
+class DistanceType(str, Enum):
+    EUCLIDEAN = "EUCLIDEAN"
+    COSINE = "COSINE"
+    DOT_PRODUCT = "DOT_PRODUCT"
+
+
+class BigQueryVectorStore(BasePydanticVectorStore):
+    """
+    Vector store index using Google BigQuery.
+
+    Provides integration with BigQuery for storing and querying vector embeddings.
+    For more information, visit: https://cloud.google.com/bigquery/docs/vector-search-intro
+
+    Required IAM Permissions:
+        - `roles/bigquery.dataOwner` (BigQuery Data Owner)
+        - `roles/bigquery.dataEditor` (BigQuery Data Editor)
+
+    Examples:
+        `pip install llama-index-vector-stores-bigquery`
+
+        ```python
+        from google.cloud.bigquery import Client
+        from llama_index.vector_stores.bigquery import BigQueryVectorStore
+
+        client = Client()
+
+        vector_store = BigQueryVectorStore(
+            table_id="my_bigquery_table",
+            dataset_id="my_bigquery_dataset",
+            bigquery_client=client,
+        )
+        ```
+
+    """
+
+    stores_text: bool = True
+    distance_type: DistanceType = DistanceType.EUCLIDEAN
+
+    _table: bigquery.Table = PrivateAttr()
+    _dataset: bigquery.Dataset = PrivateAttr()
+    _client: bigquery.Client = PrivateAttr()
+    _full_table_id: str = PrivateAttr()
+
+    def __init__(
+        self,
+        table_id: str,
+        dataset_id: str,
+        project_id: Optional[str] = None,
+        region: Optional[str] = None,
+        distance_type: Optional[DistanceType] = DistanceType.EUCLIDEAN,
+        auth_credentials: Optional[credentials.Credentials] = None,
+        bigquery_client: Optional[bigquery.Client] = None,
+        **kwargs: Any,
+    ):
+        """
+        Initialize a BigQuery Vector store.
+
+        If a `bigquery_client` is provided, it will be used directly. Otherwise, a client will be initialized using
+        the optional `project_id`, `region`, and/or `auth_credentials`. If none are provided, default credentials
+        will be used. For details on authentication, visit:
+        https://googleapis.dev/python/google-api-core/latest/auth.html
+
+        Args:
+            table_id: The ID of the BigQuery table to use for vector storage.
+            dataset_id: The ID of the dataset containing the table.
+            project_id: The GCP project ID. If not provided, it will be inferred from the client or environment.
+            region: Optionally specify a default location for datasets / tables.
+            distance_type: Optionally specify a distance type to use `EUCLIDEAN`, `COSINE`, or `DOT_PRODUCT`.
+            auth_credentials: Optional credentials object used to authenticate with BigQuery.
+            bigquery_client: An existing BigQuery client instance. If not provided, one will be created.
+            **kwargs: Additional keyword arguments passed to the parent class.
+
+        """
+        super().__init__(
+            **kwargs,
+        )
+
+        self._client: bigquery.Client = bigquery_client or self._initialize_client(
+            project_id, region, auth_credentials
+        )
+        self._dataset: bigquery.Dataset = self._create_dataset_if_not_exists(dataset_id)
+        self._table: bigquery.Table = self._create_table_if_not_exists(table_id)
+        self._full_table_id: str = (
+            f"{self._client.project}.{self._dataset.dataset_id}.{self._table.table_id}"
+        )
+        self.distance_type: DistanceType = DistanceType(distance_type)
+
+    @classmethod
+    def from_params(
+        cls,
+        table_id: str,
+        dataset_id: str,
+        project_id: Optional[str] = None,
+        region: Optional[str] = None,
+        distance_type: Optional[DistanceType] = DistanceType.EUCLIDEAN,
+        auth_credentials: Optional[credentials.Credentials] = None,
+        bigquery_client: Optional[bigquery.Client] = None,
+    ) -> "BigQueryVectorStore":
+        """
+        Initialize a BigQuery Vector store.
+
+        Args:
+            table_id: The ID of the BigQuery table to use for vector storage.
+            dataset_id: The ID of the dataset containing the table.
+            project_id: The GCP project ID. If not provided, it will be inferred from the client or environment.
+            region: Optionally specify a default location for datasets / tables.
+            distance_type: Optionally specify a distance type to use `EUCLIDEAN`, `COSINE`, or `DOT_PRODUCT`.
+            auth_credentials: Optional credentials object used to authenticate with BigQuery.
+            bigquery_client: An existing BigQuery client instance. If not provided, one will be created.
+
+        Returns:
+            BigQueryVectorStore
+
+        """
+        return cls(
+            table_id=table_id,
+            dataset_id=dataset_id,
+            project_id=project_id,
+            region=region,
+            distance_type=distance_type,
+            auth_credentials=auth_credentials,
+            bigquery_client=bigquery_client,
+        )
+
+    @property
+    def client(self) -> Union[bigquery.Client, None]:
+        """Return the BigQuery client."""
+        if not self._client:
+            return None
+        return self._client
+
+    @staticmethod
+    def _initialize_client(
+        project_id: Union[str, None],
+        region: Union[str, None],
+        auth_credentials: Union[credentials.Credentials, None],
+    ) -> bigquery.Client:
+        """
+        Initialize a new BigQuery client using the provided `project_id`, `region` and/or `auth_credentials`.
+        Defaults will be used in place of missing arguments. For details on authentication, see:
+        https://googleapis.dev/python/google-api-core/latest/auth.html
+
+        Args:
+            project_id: GCP project ID for the new client, or None to use default project resolution.
+            region: GCP region for the new client, or None to use default region.
+            auth_credentials: Credentials to authenticate the new client, or None to use default credentials.
+
+        Returns:
+            An initialized BigQuery client.
+
+        """
+        return bigquery.Client(
+            project=project_id or None,
+            location=region or None,
+            credentials=auth_credentials or None,
+        )
+
+    @staticmethod
+    def _bigquery_row_to_node(row: _BigQueryRow) -> BaseNode:
+        """
+        Convert a BigQuery row to a BaseNode object.
+
+        Args:
+            row: A row retrieved from BigQuery containing node_id, text,
+                metadata, embedding, and optional distance.
+
+        Returns:
+            Node object.
+
+        """
+        node_id: str = row.node_id
+        text: str = row.text
+        metadata: Dict[str, Any] = row.metadata
+        embedding: List[float] = row.embedding
+        _: Union[float, None] = row.distance
+
+        try:
+            node = metadata_dict_to_node(metadata)
+            node.set_content(text)
+            node.embedding = embedding
+        except (ValueError, TypeError) as e:
+            node = TextNode(
+                id_=node_id,
+                text=text,
+                metadata=metadata,
+                embedding=embedding,
+            )
+            _logger.warning(
+                f"Failed to construct node {node_id} from metadata. Falling back to manual construction. Error: {e}"
+            )
+
+        return node
+
+    def _create_dataset_if_not_exists(self, dataset_id: str) -> bigquery.Dataset:
+        """
+        Create a BigQuery dataset if it does not already exist.
+
+        For more details on creating datasets, visit:
+        https://cloud.google.com/bigquery/docs/datasets#create-dataset
+
+        Args:
+            dataset_id: The ID of the dataset to create.
+
+        Returns:
+            Dataset ID.
+
+        """
+        dataset_ref = bigquery.dataset.DatasetReference(
+            project=self._client.project, dataset_id=dataset_id
+        )
+
+        return self._client.create_dataset(dataset_ref, exists_ok=True)
+
+    def _create_table_if_not_exists(self, table_id) -> bigquery.Table:
+        """
+        Create a BigQuery table if it does not already exist.
+
+        For more information on creating tables, visit:
+        https://cloud.google.com/bigquery/docs/tables#create-table
+
+        Args:
+            table_id: The ID of the table to create.
+
+        Returns:
+            BigQuery table instance.
+
+        """
+        schema = [
+            bigquery.SchemaField("node_id", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("text", "STRING", mode="REQUIRED"),
+            bigquery.SchemaField("metadata", "JSON"),
+            bigquery.SchemaField("embedding", "FLOAT", mode="REPEATED"),
+        ]
+
+        table_ref = bigquery.TableReference.from_string(
+            f"{self._client.project}.{self._dataset.dataset_id}.{table_id}"
+        )
+        to_create = bigquery.Table(table_ref, schema=schema)
+
+        return self._client.create_table(to_create, exists_ok=True)
+
+    def add(self, nodes: List[BaseNode], **add_kwargs: Any) -> List[str]:
+        """
+        Add nodes to index.
+
+        Args:
+            nodes: List of nodes with embeddings.
+
+        Returns:
+            List of node IDs that were added.
+
+        """
+        node_ids: List[str] = []
+        json_records: List[Dict[str, Any]] = []
+
+        for node in nodes:
+            record = {
+                "node_id": node.node_id,
+                "text": node.get_content(metadata_mode=MetadataMode.NONE),
+                "embedding": node.get_embedding(),
+                "metadata": node_to_metadata_dict(
+                    node, remove_text=True, flat_metadata=False
+                ),
+            }
+            node_ids.append(node.node_id)
+            json_records.append(record)
+
+        job_config = bigquery.LoadJobConfig(schema=self._table.schema)
+        job = self._client.load_table_from_json(
+            json_rows=json_records, destination=self._table, job_config=job_config
+        )
+        job.result()
+
+        return node_ids
+
+    def delete(self, ref_doc_id: str, **delete_kwargs: Any) -> None:
+        """
+        Delete nodes using with ref_doc_id.
+
+        Args:
+            ref_doc_id : The doc_id of the document to delete.
+
+        """
+        query = f"""
+        DELETE FROM `{self._full_table_id}`
+        WHERE  SAFE.JSON_VALUE(metadata, '$."doc_id"') = @to_delete;
+        """
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=[
+                bigquery.ScalarQueryParameter(
+                    name="to_delete", type_="STRING", value=ref_doc_id
+                ),
+            ]
+        )
+
+        self._client.query_and_wait(query, job_config=job_config)
+
+    def query(self, query: VectorStoreQuery, **kwargs: Any) -> VectorStoreQueryResult:
+        """
+        Query the vector store using BigQuery's VECTOR_SEARCH to retrieve the top-k most similar nodes.
+
+        When `MetadataFilters` are provided and the table is indexed on relevant columns, BigQuery attempts to optimize
+        the search with pre-filtering before nearest neighbor search. If filters don't align with an index,
+        post-filtering is applied after similarity search, potentially returning fewer than `similarity_top_k results`.
+        Consider increasing `similarity_top_k` when post-filtering is expected.
+
+        For more information on pre-filtering and post-filtering, see:
+        https://cloud.google.com/bigquery/docs/vector-index#pre-filters_and_post-filters
+
+        Assumes embeddings are normalized for similarity scoring.
+
+        Args:
+            query: Contains the query embedding, similarity_top_k value, and optional metadata filters.
+
+        Returns:
+            VectorStoreQueryResult
+
+        """
+        where_clause, query_params = build_where_clause_and_params(
+            filters=query.filters, node_ids=query.node_ids
+        )
+
+        base_table_query = f"""
+        SELECT
+            node_id,
+            text,
+            metadata,
+            embedding
+        FROM `{self._full_table_id}`
+        """
+
+        if where_clause:
+            base_table_query += f" WHERE {where_clause}"
+
+        query_table_query = f"SELECT {query.query_embedding} AS input_embedding"
+
+        vector_search_query = f"""
+        SELECT  base.node_id   AS node_id,
+                base.text      AS text,
+                base.metadata  AS metadata,
+                base.embedding AS embedding,
+                distance
+        FROM
+            VECTOR_SEARCH(
+                ({base_table_query}), 'embedding',
+                ({query_table_query}), 'input_embedding',
+                top_k => @top_k,
+                distance_type => @distance_type
+        );
+        """
+
+        query_params.extend(
+            [
+                bigquery.ScalarQueryParameter(
+                    "top_k", type_="INTEGER", value=query.similarity_top_k
+                ),
+                bigquery.ScalarQueryParameter(
+                    "distance_type", type_="STRING", value=self.distance_type
+                ),
+            ]
+        )
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=query_params,
+        )
+        rows: bigquery.table.RowIterator = self._client.query_and_wait(
+            vector_search_query, job_config=job_config
+        )
+
+        top_k_nodes: List[BaseNode] = []
+        top_k_scores: List[float] = []
+        top_k_ids: List[str] = []
+
+        for record in rows:
+            row = _BigQueryRow(
+                node_id=record.node_id,
+                text=record.text,
+                metadata=record.metadata,
+                embedding=record.embedding,
+                distance=record.distance,
+            )
+            node = self._bigquery_row_to_node(row)
+            node_id = record.node_id
+            # Assumes embeddings are normalized.
+            score = (
+                1 / (1 + record.distance)
+                if self.distance_type == DistanceType.EUCLIDEAN
+                else (1 + record.distance) / 2
+            )
+
+            top_k_nodes.append(node)
+            top_k_scores.append(score)
+            top_k_ids.append(node_id)
+
+        return VectorStoreQueryResult(
+            nodes=top_k_nodes, similarities=top_k_scores, ids=top_k_ids
+        )
+
+    def get_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+    ) -> List[BaseNode]:
+        """
+        Retrieve nodes from BigQuery using node IDs, metadata filters, or both.
+
+        If both `node_ids` and `filters` are provided, only nodes that satisfy
+        both conditions will be returned.
+
+        Args:
+            node_ids: Optional list of node IDs for retrieval.
+            filters : Optional MetadataFilters filters for retrieval.
+
+        Returns:
+            A list of matching nodes.
+
+        Raises:
+            ValueError: If neither `node_ids` nor `filters` is provided.
+
+        """
+        if not (node_ids or filters):
+            raise ValueError(
+                "get_nodes requires at least one filtering parameter: "
+                "'node_ids', 'filters', or both. Received neither."
+            )
+
+        where_clause, query_params = build_where_clause_and_params(node_ids, filters)
+
+        query = f"""
+        SELECT  node_id,
+                text,
+                embedding,
+                metadata
+        FROM    `{self._full_table_id}`
+        WHERE   {where_clause};
+        """
+
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=query_params,
+        )
+        rows: bigquery.table.RowIterator = self._client.query_and_wait(
+            query, job_config=job_config
+        )
+
+        nodes: List[BaseNode] = []
+        for record in rows:
+            row = _BigQueryRow(
+                node_id=record.node_id,
+                text=record.text,
+                metadata=record.metadata,
+                embedding=record.embedding,
+                distance=record.distance,
+            )
+            node = self._bigquery_row_to_node(row)
+
+            nodes.append(node)
+
+        return nodes
+
+    def delete_nodes(
+        self,
+        node_ids: Optional[List[str]] = None,
+        filters: Optional[MetadataFilters] = None,
+        **delete_kwargs: Any,
+    ) -> None:
+        """
+        Delete nodes from BigQuery based on node IDs, metadata filters, or both.
+
+        If both `node_ids` and `filters` are provided, only nodes matching both
+        criteria will be deleted.
+
+        Args:
+            node_ids: Optional list of node IDs to delete.
+            filters : Optional MetadataFilters filters for deletion.
+
+        Raises:
+            ValueError: If neither `node_ids` nor `filters` are provided.
+
+        """
+        if not (node_ids or filters):
+            raise ValueError(
+                "delete_nodes requires at least one filtering parameter: "
+                "'node_ids', 'filters', or both. Received neither."
+            )
+
+        where_clause, query_params = build_where_clause_and_params(node_ids, filters)
+
+        query = f"""
+        DELETE FROM `{self._full_table_id}`
+        WHERE {where_clause};
+        """
+
+        job_config = bigquery.QueryJobConfig(
+            query_parameters=query_params,
+        )
+        self._client.query_and_wait(query, job_config=job_config)
+
+    def clear(self) -> None:
+        """
+        Clears the index.
+
+        This truncates the underlying table in BigQuery.
+        """
+        query = f"""TRUNCATE TABLE `{self._full_table_id}`;"""
+        self._client.query_and_wait(query)

llama_index_vector_stores_bigquery-0.1.0/llama_index/vector_stores/bigquery/utils.py

@@ -0,0 +1,162 @@
+from typing import List, Optional, Tuple, Union
+
+from google.cloud import bigquery
+from llama_index.core.vector_stores.types import (
+    FilterOperator,
+    MetadataFilter,
+    MetadataFilters,
+)
+
+
+def build_where_clause_and_params(
+    node_ids: Optional[List[str]] = None,
+    filters: Optional[MetadataFilters] = None,
+) -> Tuple[
+    str, List[Union[bigquery.ScalarQueryParameter, bigquery.ArrayQueryParameter]]
+]:
+    """
+    Construct a parameterized SQL WHERE clause and corresponding query parameters.
+
+    The clause is built from the provided node IDs and metadata filters. Parameters
+    are returned separately to support safe, parameterized queries in BigQuery,
+    helping to prevent SQL injection. See:
+    https://cloud.google.com/bigquery/docs/parameterized-queries
+
+    If both `node_ids` and `filters` are provided, the resulting WHERE clause
+    combines conditions using AND logic.
+
+    Args:
+        node_ids: A list of node IDs to include in the filter.
+        filters: Metadata filters to apply to the query.
+
+    Returns:
+        A tuple (where_clause, query_params), where where_clause is a SQL WHERE clause string,
+        and query_params is a list of query parameters to bind to the query.
+
+    """
+    query_params: List[
+        Union[bigquery.ScalarQueryParameter, bigquery.ArrayQueryParameter]
+    ] = []
+    conditions: List[str] = []
+
+    if filters:
+        filter_where_clause, filter_query_params = (
+            _recursive_build_where_clause_from_filters(filters)
+        )
+        conditions.append(filter_where_clause)
+        query_params.extend(filter_query_params)
+
+    if node_ids:
+        conditions.append("node_id IN UNNEST(@node_ids)")
+        query_params.append(
+            bigquery.ArrayQueryParameter(
+                name="node_ids", array_type="STRING", values=node_ids
+            )
+        )
+
+    # if both `node_ids` and `filters` are provided, both criteria should be considered
+    where_clause = " AND ".join(conditions)
+
+    return where_clause, query_params
+
+
+def _recursive_build_where_clause_from_filters(
+    meta_filters: MetadataFilters,
+) -> Tuple[
+    str, List[Union[bigquery.ScalarQueryParameter, bigquery.ArrayQueryParameter]]
+]:
+    """
+    Recursively construct a SQL WHERE clause and corresponding query parameters
+
+    The provided MetadataFilters object may contain nested filter groups. This function
+    traverses them recursively to build a complete WHERE clause and the associated
+    query parameters for use in a parameterized BigQuery query.
+
+    Args:
+        meta_filters: A potentially nested MetadataFilters filter object
+
+    Returns:
+        A tuple (where_clause, query_params), where where_clause is a parameterized SQL WHERE clause string,
+        and query_params is a list of query parameters to bind to the query.
+
+    """
+    filters_list: List[str] = []
+    query_params: List[
+        Union[bigquery.ScalarQueryParameter, bigquery.ArrayQueryParameter]
+    ] = []
+
+    for filter_ in meta_filters.filters:
+        clause, params = (
+            _recursive_build_where_clause_from_filters(filter_)
+            if isinstance(filter_, MetadataFilters)
+            else _build_filter_clause(filter_)
+        )
+
+        filters_list.append(clause)
+        query_params.extend(params)
+
+    condition = f" {meta_filters.condition.value.upper()} "
+    filters_ = f"({condition.join(filters_list)})"
+
+    return filters_, query_params
+
+
+def _build_filter_clause(
+    filter_: MetadataFilter,
+) -> Tuple[
+    str, List[Union[bigquery.ScalarQueryParameter, bigquery.ArrayQueryParameter]]
+]:
+    field = filter_.key
+    operator = filter_.operator
+    value = filter_.value
+
+    if operator == FilterOperator.IS_EMPTY:
+        clause = f"JSON_TYPE(JSON_QUERY(metadata, '$.\"{field}\"')) = 'null'"
+        params = []
+    elif operator == FilterOperator.IN or operator == FilterOperator.NIN:
+        bigquery_operator = _llama_to_bigquery_operator(operator)
+        clause = (
+            f"SAFE.JSON_VALUE(metadata, '$.\"{field}\"') {bigquery_operator} UNNEST(?)"
+        )
+        params = [
+            bigquery.ArrayQueryParameter(name=None, array_type="STRING", values=value)
+        ]
+    elif operator == FilterOperator.TEXT_MATCH:
+        bigquery_operator = _llama_to_bigquery_operator(operator)
+        clause = (
+            f"SAFE.JSON_VALUE(metadata, '$.\"{field}\"') {bigquery_operator} '{value}'"
+        )
+        params = [bigquery.ScalarQueryParameter(name=None, type_="STRING", value=value)]
+    elif operator == FilterOperator.TEXT_MATCH_INSENSITIVE:
+        bigquery_operator = _llama_to_bigquery_operator(operator)
+        clause = f"LOWER(SAFE.JSON_VALUE(metadata, '$.\"{field}\"')) {bigquery_operator} LOWER('{value}')"
+        params = [bigquery.ScalarQueryParameter(name=None, type_="STRING", value=value)]
+    else:
+        bigquery_operator = _llama_to_bigquery_operator(operator)
+        clause = f"SAFE.JSON_VALUE(metadata, '$.\"{field}\"') {bigquery_operator} ?"
+        params = [bigquery.ScalarQueryParameter(name=None, type_="STRING", value=value)]
+
+    return clause, params
+
+
+def _llama_to_bigquery_operator(operator: FilterOperator) -> str:
+    operator_map = {
+        FilterOperator.EQ: "=",
+        FilterOperator.GT: ">",
+        FilterOperator.LT: "<",
+        FilterOperator.NE: "!=",
+        FilterOperator.GTE: ">=",
+        FilterOperator.LTE: "<=",
+        FilterOperator.IN: "IN",
+        FilterOperator.NIN: "NOT IN",
+        FilterOperator.TEXT_MATCH: "LIKE",
+        FilterOperator.TEXT_MATCH_INSENSITIVE: "LIKE",
+        FilterOperator.IS_EMPTY: "IS NULL",
+    }
+
+    try:
+        return operator_map[operator]
+    except KeyError:
+        raise ValueError(
+            f"Invalid operator `{operator.value}` is not a supported BigQuery operator."
+        )

llama_index_vector_stores_bigquery-0.1.0/pyproject.toml

@@ -0,0 +1,64 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[dependency-groups]
+dev = [
+    "ipython==8.10.0",
+    "jupyter>=1.0.0,<2",
+    "mypy==0.991",
+    "pre-commit==3.2.0",
+    "pylint==2.15.10",
+    "pytest==7.2.1",
+    "pytest-mock==3.11.1",
+    "ruff==0.11.11",
+    "types-Deprecated>=0.1.0",
+    "types-PyYAML>=6.0.12.12,<7",
+    "types-protobuf>=4.24.0.4,<5",
+    "types-redis==4.5.5.0",
+    "types-requests==2.28.11.8",
+    "types-setuptools==67.1.0.0",
+    "black[jupyter]<=23.9.1,>=23.7.0",
+    "codespell[toml]>=v2.2.6",
+    "diff-cover>=9.2.0",
+    "pytest-cov>=6.1.1",
+    "sqlparse>=0.5.3",
+]
+
+[project]
+name = "llama-index-vector-stores-bigquery"
+version = "0.1.0"
+description = "llama-index vector_stores bigquery vector search integration"
+requires-python = ">=3.9,<4.0"
+readme = "README.md"
+license = "MIT"
+dependencies = [
+    "google-cloud-bigquery>=3.34.0,<4",
+    "llama-index-core>=0.13.0,<0.14",
+]
+
+[tool.codespell]
+check-filenames = true
+check-hidden = true
+skip = "*.csv,*.html,*.json,*.jsonl,*.pdf,*.txt,*.ipynb"
+
+[tool.hatch.build.targets.sdist]
+include = ["llama_index/"]
+exclude = ["**/BUILD"]
+
+[tool.hatch.build.targets.wheel]
+include = ["llama_index/"]
+exclude = ["**/BUILD"]
+
+[tool.llamahub]
+contains_example = false
+import_path = "llama_index.vector_stores.bigquery"
+
+[tool.llamahub.class_authors]
+BigQueryVectorStore = "llama-index"
+
+[tool.mypy]
+disallow_untyped_defs = true
+exclude = ["_static", "build", "examples", "notebooks", "venv"]
+ignore_missing_imports = true
+python_version = "3.8"