local-bigquery 0.3.0__tar.gz

local_bigquery-0.3.0/.gitignore

@@ -0,0 +1,6 @@
+.idea
+.venv
+__pycache__
+*.db
+.claude/
+.hypothesis/

local_bigquery-0.3.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 The Authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

local_bigquery-0.3.0/PKG-INFO

@@ -0,0 +1,168 @@
+Metadata-Version: 2.5
+Name: local-bigquery
+Version: 0.3.0
+Summary: A BigQuery emulator for local development and tests
+Project-URL: Repository, https://github.com/novucs/local-bigquery
+Project-URL: Issues, https://github.com/novucs/local-bigquery/issues
+License-Expression: MIT
+License-File: LICENSE
+Requires-Python: <3.14,>=3.13
+Requires-Dist: duckdb>=1.5.5
+Requires-Dist: fastapi>=0.139
+Requires-Dist: fastavro>=1.12.2
+Requires-Dist: google-cloud-bigquery-storage>=2.41.0
+Requires-Dist: grpcio>=1.83.0
+Requires-Dist: mini-racer>=0.12.4
+Requires-Dist: prompt-toolkit>=3.0.50
+Requires-Dist: pyarrow>=25.0.1
+Requires-Dist: pydantic-settings>=2.8.1
+Requires-Dist: pygments>=2.19.1
+Requires-Dist: sqlglot>=30.19
+Requires-Dist: uvicorn>=0.35
+Description-Content-Type: text/markdown
+
+# Local BigQuery
+
+A BigQuery emulator for development and tests. It serves the BigQuery REST and Storage
+APIs and runs GoogleSQL on DuckDB, so the official clients work with no project or
+credentials.
+
+## Quick start
+
+```bash
+docker run -p 9050:9050 -p 9060:9060 -v bigquery:/data ghcr.io/novucs/local-bigquery:0.3.0
+```
+
+Then point a client at `http://localhost:9050`, without credentials. Any project ID
+works, and data persists in `/data`.
+
+## Connecting
+
+### Python
+
+```python
+from google.auth.credentials import AnonymousCredentials
+from google.cloud import bigquery
+
+client = bigquery.Client(
+    project="local",
+    credentials=AnonymousCredentials(),
+    client_options={"api_endpoint": "http://localhost:9050"},
+)
+rows = client.query_and_wait("SELECT 1 AS x")
+```
+
+For faster `to_dataframe()`, add the Storage Read API, served as plain gRPC on port `9060`:
+
+```python
+import grpc
+from google.cloud import bigquery_storage_v1
+from google.cloud.bigquery_storage_v1.services.big_query_read.transports import (
+    BigQueryReadGrpcTransport,
+)
+
+transport = BigQueryReadGrpcTransport(channel=grpc.insecure_channel("localhost:9060"))
+bqstorage = bigquery_storage_v1.BigQueryReadClient(transport=transport)
+frame = client.query_and_wait("SELECT 1 AS x").to_dataframe(bqstorage_client=bqstorage)
+```
+
+For SQLAlchemy, pass the client in:
+`create_engine("bigquery://local", connect_args={"client": client})`.
+
+### Go
+
+```go
+client, err := bigquery.NewClient(ctx, "local",
+    option.WithEndpoint("http://localhost:9050/bigquery/v2/"),
+    option.WithoutAuthentication(),
+)
+```
+
+### Node.js
+
+```js
+import { BigQuery } from "@google-cloud/bigquery";
+
+const bigquery = new BigQuery({ projectId: "local", apiEndpoint: "http://localhost:9050" });
+const [rows] = await bigquery.query("SELECT 1 AS x");
+```
+
+## Testing with pytest
+
+```bash
+pip install local-bigquery
+```
+
+This adds pytest fixtures that run the emulator inside the test process. There's no
+Docker, and it starts in under a second:
+
+- `bigquery_client`: a client connected to an emulator shared by the whole test run.
+- `bigquery_dataset`: a new dataset for each test, deleted afterwards.
+- `bigquery_emulator`: the emulator's `rest_url`, `grpc_address` and `project_id`.
+
+```python
+def test_totals(bigquery_client, bigquery_dataset):
+    table = f"{bigquery_dataset.dataset_id}.orders"
+    bigquery_client.query_and_wait(f"CREATE TABLE {table} (customer STRING, amount INT64)")
+    bigquery_client.insert_rows_json(table, [{"customer": "a", "amount": 2}] * 3)
+    rows = bigquery_client.query_and_wait(f"SELECT SUM(amount) AS total FROM {table}")
+    assert [row.total for row in rows] == [6]
+```
+
+## Features
+
+- **SQL**: GoogleSQL functions, DML including `MERGE`, DDL, scripting, procedures,
+  SQL/JavaScript UDFs, table functions, wildcard tables, `INFORMATION_SCHEMA`.
+- **Tables**: partitioning, clustering, constraints, views, materialized views,
+  snapshots, clones, time travel, search and vector indexes.
+- **Jobs**: query, load, copy and extract, with dry runs, cancellation and sessions.
+- **Data**: CSV, JSON, Parquet, Avro and ORC loads, and CSV, JSON, Parquet and Avro exports.
+  Also `insert_rows_json` and the Storage Read and Write APIs.
+- **`gs://`**: maps to `$DATA_DIR/gcs`, or to a GCS emulator set by `STORAGE_EMULATOR_HOST`.
+- **Row access policies**: the caller is the service account that signed the request.
+  Signatures aren't checked, so any key works:
+  `service_account.Credentials.from_service_account_file("key.json", always_use_jwt_access=True)`.
+  Unsigned requests run as `CALLER`.
+- **`EXTERNAL_QUERY`** against Postgres at `POSTGRES_URI`.
+- **REPL**: `local-bigquery repl`. Delete all data with `local-bigquery reset`.
+
+## Limitations
+
+- For test-sized data on one machine. There's no authentication, and IAM policies
+  aren't enforced.
+- Legacy SQL and BigQuery ML functions (`ML.PREDICT` and so on) aren't supported.
+- A failed statement aborts the whole transaction.
+- `STRING(n)` and `BYTES(n)` lengths aren't enforced.
+- Other API methods return `501`.
+
+Found a difference from BigQuery? [Open an issue](https://github.com/novucs/local-bigquery/issues).
+
+## Configuration
+
+Set these as environment variables, or pass the matching flag to `local-bigquery`.
+
+| Variable | Flag | Default |
+|---|---|---|
+| `BIGQUERY_PORT` | `--port` | `9050` |
+| `GRPC_PORT` | `--grpc-port` | `9060` |
+| `DATA_DIR` | `--data-dir` | `/data` |
+| `DEFAULT_PROJECT_ID` | `--project` | `local` |
+| `DEFAULT_DATASET_ID` | `--dataset` | `local` |
+| `CALLER` | | `user:local-bigquery@localhost` |
+| `GROUPS` | | `{}`, e.g. `{"user:alice@example.com": ["group:team@example.com"]}` |
+| `GCS_LOCAL_ROOT` | | `$DATA_DIR/gcs` |
+| `STORAGE_EMULATOR_HOST` | | |
+| `POSTGRES_CONNECTION_ID` | | `us.default` |
+| `POSTGRES_URI` | | `postgresql://postgres:example@db:5432/postgres` |
+
+## Development
+
+```bash
+uv run local-bigquery --data-dir /tmp/local-bigquery   # run from source
+uv run pytest                                           # run the tests
+uv run pytest --endpoint google --project <project>     # run them against real BigQuery
+```
+
+## License
+
+[MIT](LICENSE)

local_bigquery-0.3.0/README.md

@@ -0,0 +1,145 @@
+# Local BigQuery
+
+A BigQuery emulator for development and tests. It serves the BigQuery REST and Storage
+APIs and runs GoogleSQL on DuckDB, so the official clients work with no project or
+credentials.
+
+## Quick start
+
+```bash
+docker run -p 9050:9050 -p 9060:9060 -v bigquery:/data ghcr.io/novucs/local-bigquery:0.3.0
+```
+
+Then point a client at `http://localhost:9050`, without credentials. Any project ID
+works, and data persists in `/data`.
+
+## Connecting
+
+### Python
+
+```python
+from google.auth.credentials import AnonymousCredentials
+from google.cloud import bigquery
+
+client = bigquery.Client(
+    project="local",
+    credentials=AnonymousCredentials(),
+    client_options={"api_endpoint": "http://localhost:9050"},
+)
+rows = client.query_and_wait("SELECT 1 AS x")
+```
+
+For faster `to_dataframe()`, add the Storage Read API, served as plain gRPC on port `9060`:
+
+```python
+import grpc
+from google.cloud import bigquery_storage_v1
+from google.cloud.bigquery_storage_v1.services.big_query_read.transports import (
+    BigQueryReadGrpcTransport,
+)
+
+transport = BigQueryReadGrpcTransport(channel=grpc.insecure_channel("localhost:9060"))
+bqstorage = bigquery_storage_v1.BigQueryReadClient(transport=transport)
+frame = client.query_and_wait("SELECT 1 AS x").to_dataframe(bqstorage_client=bqstorage)
+```
+
+For SQLAlchemy, pass the client in:
+`create_engine("bigquery://local", connect_args={"client": client})`.
+
+### Go
+
+```go
+client, err := bigquery.NewClient(ctx, "local",
+    option.WithEndpoint("http://localhost:9050/bigquery/v2/"),
+    option.WithoutAuthentication(),
+)
+```
+
+### Node.js
+
+```js
+import { BigQuery } from "@google-cloud/bigquery";
+
+const bigquery = new BigQuery({ projectId: "local", apiEndpoint: "http://localhost:9050" });
+const [rows] = await bigquery.query("SELECT 1 AS x");
+```
+
+## Testing with pytest
+
+```bash
+pip install local-bigquery
+```
+
+This adds pytest fixtures that run the emulator inside the test process. There's no
+Docker, and it starts in under a second:
+
+- `bigquery_client`: a client connected to an emulator shared by the whole test run.
+- `bigquery_dataset`: a new dataset for each test, deleted afterwards.
+- `bigquery_emulator`: the emulator's `rest_url`, `grpc_address` and `project_id`.
+
+```python
+def test_totals(bigquery_client, bigquery_dataset):
+    table = f"{bigquery_dataset.dataset_id}.orders"
+    bigquery_client.query_and_wait(f"CREATE TABLE {table} (customer STRING, amount INT64)")
+    bigquery_client.insert_rows_json(table, [{"customer": "a", "amount": 2}] * 3)
+    rows = bigquery_client.query_and_wait(f"SELECT SUM(amount) AS total FROM {table}")
+    assert [row.total for row in rows] == [6]
+```
+
+## Features
+
+- **SQL**: GoogleSQL functions, DML including `MERGE`, DDL, scripting, procedures,
+  SQL/JavaScript UDFs, table functions, wildcard tables, `INFORMATION_SCHEMA`.
+- **Tables**: partitioning, clustering, constraints, views, materialized views,
+  snapshots, clones, time travel, search and vector indexes.
+- **Jobs**: query, load, copy and extract, with dry runs, cancellation and sessions.
+- **Data**: CSV, JSON, Parquet, Avro and ORC loads, and CSV, JSON, Parquet and Avro exports.
+  Also `insert_rows_json` and the Storage Read and Write APIs.
+- **`gs://`**: maps to `$DATA_DIR/gcs`, or to a GCS emulator set by `STORAGE_EMULATOR_HOST`.
+- **Row access policies**: the caller is the service account that signed the request.
+  Signatures aren't checked, so any key works:
+  `service_account.Credentials.from_service_account_file("key.json", always_use_jwt_access=True)`.
+  Unsigned requests run as `CALLER`.
+- **`EXTERNAL_QUERY`** against Postgres at `POSTGRES_URI`.
+- **REPL**: `local-bigquery repl`. Delete all data with `local-bigquery reset`.
+
+## Limitations
+
+- For test-sized data on one machine. There's no authentication, and IAM policies
+  aren't enforced.
+- Legacy SQL and BigQuery ML functions (`ML.PREDICT` and so on) aren't supported.
+- A failed statement aborts the whole transaction.
+- `STRING(n)` and `BYTES(n)` lengths aren't enforced.
+- Other API methods return `501`.
+
+Found a difference from BigQuery? [Open an issue](https://github.com/novucs/local-bigquery/issues).
+
+## Configuration
+
+Set these as environment variables, or pass the matching flag to `local-bigquery`.
+
+| Variable | Flag | Default |
+|---|---|---|
+| `BIGQUERY_PORT` | `--port` | `9050` |
+| `GRPC_PORT` | `--grpc-port` | `9060` |
+| `DATA_DIR` | `--data-dir` | `/data` |
+| `DEFAULT_PROJECT_ID` | `--project` | `local` |
+| `DEFAULT_DATASET_ID` | `--dataset` | `local` |
+| `CALLER` | | `user:local-bigquery@localhost` |
+| `GROUPS` | | `{}`, e.g. `{"user:alice@example.com": ["group:team@example.com"]}` |
+| `GCS_LOCAL_ROOT` | | `$DATA_DIR/gcs` |
+| `STORAGE_EMULATOR_HOST` | | |
+| `POSTGRES_CONNECTION_ID` | | `us.default` |
+| `POSTGRES_URI` | | `postgresql://postgres:example@db:5432/postgres` |
+
+## Development
+
+```bash
+uv run local-bigquery --data-dir /tmp/local-bigquery   # run from source
+uv run pytest                                           # run the tests
+uv run pytest --endpoint google --project <project>     # run them against real BigQuery
+```
+
+## License
+
+[MIT](LICENSE)

local_bigquery-0.3.0/pyproject.toml

@@ -0,0 +1,60 @@
+[project]
+name = "local-bigquery"
+version = "0.3.0"
+description = "A BigQuery emulator for local development and tests"
+readme = "README.md"
+license = "MIT"
+license-files = ["LICENSE"]
+requires-python = ">=3.13,<3.14"
+dependencies = [
+    "duckdb>=1.5.5",
+    "fastapi>=0.139",
+    "fastavro>=1.12.2",
+    "google-cloud-bigquery-storage>=2.41.0",
+    "grpcio>=1.83.0",
+    "mini-racer>=0.12.4",
+    "prompt-toolkit>=3.0.50",
+    "pyarrow>=25.0.1",
+    "pydantic-settings>=2.8.1",
+    "pygments>=2.19.1",
+    "sqlglot>=30.19",
+    "uvicorn>=0.35",
+]
+
+[project.urls]
+Repository = "https://github.com/novucs/local-bigquery"
+Issues = "https://github.com/novucs/local-bigquery/issues"
+
+[project.scripts]
+local-bigquery = "local_bigquery.cli:main"
+
+[project.entry-points.pytest11]
+local_bigquery = "local_bigquery.pytest_plugin"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.sdist]
+include = ["src"]
+
+[dependency-groups]
+dev = [
+    "cryptography>=49.0.0",
+    "datamodel-code-generator>=0.82.0",
+    "db-dtypes>=1.7.1",
+    "google-cloud-bigquery>=3.31.0",
+    "hypothesis>=6.168.1",
+    "pandas>=3.0.6",
+    "psycopg2-binary>=2.9.13",
+    "pytest>=8.3.5",
+    "pytest-timeout>=2.4.0",
+    "pytest-xdist>=3.8.0",
+    "sqlalchemy-bigquery>=1.13.0",
+    "testcontainers>=4.10.0",
+]
+
+[tool.pytest.ini_options]
+xfail_strict = true
+addopts = "-p no:warnings -n auto --dist loadgroup"
+timeout = 30

local_bigquery-0.3.0/src/local_bigquery/__init__.py

@@ -0,0 +1,3 @@
+from local_bigquery.app import app
+
+__all__ = ["app"]

local_bigquery-0.3.0/src/local_bigquery/api/__init__.py

@@ -0,0 +1,30 @@
+import json
+
+from fastapi import APIRouter, Response
+
+from local_bigquery.resource import Resource
+
+
+class Router(APIRouter):
+    def add_api_route(self, path, endpoint, **kwargs):
+        super().add_api_route(
+            path, endpoint, **kwargs | {"response_model_exclude_unset": True}
+        )
+
+
+def paginate(
+    items: list, max_results: int | None, page_token: str | None
+) -> tuple[list, str | None]:
+    start = int(page_token or 0)
+    end = len(items) if max_results is None else start + max_results
+    return items[start:end], str(end) if end < len(items) else None
+
+
+def with_rows(payload: dict, rows: list[str]) -> Response:
+    body = json.dumps(
+        {key: value for key, value in payload.items() if value is not None},
+        default=Resource.dump,
+    )
+    if rows:
+        body = f'{body[:-1]}, "rows": [{",".join(rows)}]}}'
+    return Response(body, media_type="application/json")

local_bigquery-0.3.0/src/local_bigquery/api/datasets.py

@@ -0,0 +1,59 @@
+from fastapi import Header
+
+from local_bigquery.api import Router, paginate
+from local_bigquery.catalog import datasets
+from local_bigquery.models import Dataset, DatasetList, DatasetListDatasetsItem
+
+router = Router(tags=["datasets"])
+SUMMARY_FIELDS = set(DatasetListDatasetsItem.model_fields)
+
+
+@router.get("/projects/{project_id}/datasets")
+def list_datasets(
+    project_id: str,
+    maxResults: int | None = None,
+    pageToken: str | None = None,
+    all: bool = False,
+    filter: str | None = None,
+) -> DatasetList:
+    summaries = [
+        dataset.model_dump(include=SUMMARY_FIELDS, exclude_none=True)
+        for dataset in datasets.list_(project_id, filter, all)
+    ]
+    page, token = paginate(summaries, maxResults, pageToken)
+    return DatasetList(kind="bigquery#datasetList", datasets=page, nextPageToken=token)
+
+
+@router.post("/projects/{project_id}/datasets")
+def insert_dataset(project_id: str, body: Dataset) -> Dataset:
+    return datasets.create(project_id, body)
+
+
+@router.get("/projects/{project_id}/datasets/{dataset_id}")
+def get_dataset(project_id: str, dataset_id: str) -> Dataset:
+    return datasets.get(project_id, dataset_id)
+
+
+@router.patch("/projects/{project_id}/datasets/{dataset_id}")
+def patch_dataset(
+    project_id: str,
+    dataset_id: str,
+    body: Dataset,
+    if_match: str | None = Header(None),
+) -> Dataset:
+    return datasets.update(project_id, dataset_id, body, if_match, replace=False)
+
+
+@router.put("/projects/{project_id}/datasets/{dataset_id}")
+def update_dataset(
+    project_id: str,
+    dataset_id: str,
+    body: Dataset,
+    if_match: str | None = Header(None),
+) -> Dataset:
+    return datasets.update(project_id, dataset_id, body, if_match, replace=True)
+
+
+@router.delete("/projects/{project_id}/datasets/{dataset_id}", status_code=204)
+def delete_dataset(project_id: str, dataset_id: str, deleteContents: bool = False):
+    datasets.delete(project_id, dataset_id, deleteContents)

local_bigquery-0.3.0/src/local_bigquery/api/iam.py

@@ -0,0 +1,29 @@
+from local_bigquery.api import Router
+from local_bigquery.catalog import iam
+from local_bigquery.models import (
+    Policy,
+    SetIamPolicyRequest,
+    TestIamPermissionsRequest,
+    TestIamPermissionsResponse,
+)
+
+router = Router(tags=["iam"])
+RESOURCE = "/{resource:path}"
+
+
+@router.post(f"{RESOURCE}:getIamPolicy")
+def get_policy(resource: str) -> Policy:
+    return iam.get(resource)
+
+
+@router.post(f"{RESOURCE}:setIamPolicy")
+def set_policy(resource: str, body: SetIamPolicyRequest) -> Policy:
+    return iam.set_(resource, body.policy or Policy())
+
+
+@router.post(f"{RESOURCE}:testIamPermissions")
+def test_permissions(
+    resource: str, body: TestIamPermissionsRequest
+) -> TestIamPermissionsResponse:
+    iam.get(resource)
+    return TestIamPermissionsResponse(permissions=body.permissions or [])