n6k 0.1.0__tar.gz

n6k-0.1.0/.claude/settings.local.json

@@ -0,0 +1,9 @@
+{
+  "permissions": {
+    "allow": [
+      "WebSearch",
+      "WebFetch(domain:arrow.apache.org)",
+      "Bash(uv run pytest:*)"
+    ]
+  }
+}

n6k-0.1.0/.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

n6k-0.1.0/.gitignore

@@ -0,0 +1 @@
+__pycache__/

n6k-0.1.0/.python-version

@@ -0,0 +1 @@
+3.12

n6k-0.1.0/PKG-INFO

@@ -0,0 +1,9 @@
+Metadata-Version: 2.4
+Name: n6k
+Version: 0.1.0
+Summary: Arrow/Parquet REST server
+Requires-Python: >=3.12
+Requires-Dist: fastapi>=0.115
+Requires-Dist: pandas>=2
+Requires-Dist: pyarrow>=19
+Requires-Dist: uvicorn[standard]>=0.34

n6k-0.1.0/demo/app.py

@@ -0,0 +1,26 @@
+import pandas as pd
+import seaborn as sns
+from starlette.responses import PlainTextResponse
+
+import n6k_server
+
+app = n6k_server.create_app()
+
+
+@n6k_server.data(app, "demo")
+def demo():
+    return pd.DataFrame({
+        "id": [1, 2, 3],
+        "name": ["alice", "bob", "charlie"],
+        "score": [95.5, 87.3, 91.8],
+    })
+
+
+@n6k_server.data(app, "iris")
+def iris():
+    return sns.load_dataset("iris")
+
+
+@app.get("/health")
+async def health() -> PlainTextResponse:
+    return PlainTextResponse("ok")

n6k-0.1.0/frontend.md

@@ -0,0 +1,30 @@
+# Data Protocol
+
+All data responses are binary. No JSON.
+
+## Endpoints
+
+| Method | Path | Description |
+|--------|------|-------------|
+| GET | `/health` | Plain text `ok` |
+| GET | `/api/demo` | Table: `id` (int64), `name` (utf8), `score` (float64) |
+
+## Content Negotiation
+
+Set the `Accept` header to choose the wire format:
+
+| Accept | Format |
+|--------|--------|
+| `application/vnd.apache.arrow.stream` | Arrow IPC Stream |
+| `application/vnd.apache.arrow.file` | Arrow IPC File |
+| `application/x-parquet` | Parquet |
+
+Default (missing or unrecognized Accept): **Arrow IPC Stream**.
+
+First matching media type wins. Quality parameters are stripped, not weighted.
+
+The response `Content-Type` will match the negotiated format.
+
+## Errors
+
+All errors are `text/plain`. No JSON error envelopes.

n6k-0.1.0/pyproject.toml

@@ -0,0 +1,28 @@
+[project]
+name = "n6k"
+version = "0.1.0"
+description = "Arrow/Parquet REST server"
+requires-python = ">=3.12"
+dependencies = [
+    "fastapi>=0.115",
+    "uvicorn[standard]>=0.34",
+    "pyarrow>=19",
+    "pandas>=2",
+]
+
+[dependency-groups]
+dev = [
+    "httpx>=0.28",
+    "pytest>=8",
+    "pytest-asyncio>=0.25",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/n6k_server"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"

n6k-0.1.0/src/n6k_server/__init__.py

@@ -0,0 +1 @@
+from n6k_server.app import create_app, data

n6k-0.1.0/src/n6k_server/app.py

@@ -0,0 +1,44 @@
+import pandas as pd
+import pyarrow as pa
+from fastapi import FastAPI, Request
+from fastapi.middleware.cors import CORSMiddleware
+from starlette.exceptions import HTTPException as StarletteHTTPException
+from starlette.responses import PlainTextResponse
+
+from n6k_server.routes import tables
+
+
+def create_app() -> FastAPI:
+    application = FastAPI()
+    application.state.data_sources = {}
+
+    application.add_middleware(
+        CORSMiddleware,
+        allow_origins=["*"],
+        allow_methods=["*"],
+        allow_headers=["*"],
+    )
+
+    @application.exception_handler(StarletteHTTPException)
+    async def http_exception_handler(
+        request: Request, exc: StarletteHTTPException
+    ) -> PlainTextResponse:
+        return PlainTextResponse(exc.detail, status_code=exc.status_code)
+
+    @application.exception_handler(Exception)
+    async def plain_text_exception_handler(
+        request: Request, exc: Exception
+    ) -> PlainTextResponse:
+        status = getattr(exc, "status_code", 500)
+        return PlainTextResponse(str(exc), status_code=status)
+
+    application.include_router(tables.router, prefix="/tables")
+
+    return application
+
+
+def data(app: FastAPI, name: str):
+    def decorator(fn):
+        app.state.data_sources[name] = fn
+        return fn
+    return decorator

n6k-0.1.0/src/n6k_server/formats.py

@@ -0,0 +1,58 @@
+from enum import StrEnum
+from io import BytesIO
+
+import pyarrow as pa
+import pyarrow.ipc as ipc
+import pyarrow.parquet as pq
+from starlette.requests import Request
+from starlette.responses import Response
+
+
+class ArrowFormat(StrEnum):
+    IPC_STREAM = "application/vnd.apache.arrow.stream"
+    IPC_FILE = "application/vnd.apache.arrow.file"
+    PARQUET = "application/x-parquet"
+
+
+_ACCEPT_MAP: dict[str, ArrowFormat] = {f.value: f for f in ArrowFormat}
+
+
+def negotiate(request: Request) -> ArrowFormat:
+    accept = request.headers.get("accept", "")
+    for media_type in accept.split(","):
+        key = media_type.strip().split(";")[0].strip()
+        if key in _ACCEPT_MAP:
+            return _ACCEPT_MAP[key]
+    return ArrowFormat.IPC_STREAM
+
+
+def _serialize_ipc_stream(table: pa.Table) -> bytes:
+    sink = pa.BufferOutputStream()
+    with ipc.new_stream(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+def _serialize_ipc_file(table: pa.Table) -> bytes:
+    sink = pa.BufferOutputStream()
+    with ipc.new_file(sink, table.schema) as writer:
+        writer.write_table(table)
+    return sink.getvalue().to_pybytes()
+
+
+def _serialize_parquet(table: pa.Table) -> bytes:
+    buf = BytesIO()
+    pq.write_table(table, buf)
+    return buf.getvalue()
+
+
+_SERIALIZERS = {
+    ArrowFormat.IPC_STREAM: _serialize_ipc_stream,
+    ArrowFormat.IPC_FILE: _serialize_ipc_file,
+    ArrowFormat.PARQUET: _serialize_parquet,
+}
+
+
+def arrow_response(table: pa.Table, fmt: ArrowFormat) -> Response:
+    body = _SERIALIZERS[fmt](table)
+    return Response(content=body, media_type=fmt.value)

n6k-0.1.0/src/n6k_server/pushdown.py

@@ -0,0 +1,70 @@
+import json
+
+import pyarrow as pa
+import pyarrow.compute as pc
+
+
+def apply_columns(table: pa.Table, columns_param: str | None) -> pa.Table:
+    if not columns_param:
+        return table
+    names = [c.strip() for c in columns_param.split(",") if c.strip()]
+    valid = [n for n in names if n in table.schema.names]
+    if not valid:
+        return table
+    return table.select(valid)
+
+
+_OPS = {
+    "=": pc.equal,
+    "!=": pc.not_equal,
+    ">": pc.greater,
+    ">=": pc.greater_equal,
+    "<": pc.less,
+    "<=": pc.less_equal,
+}
+
+
+def apply_filters(table: pa.Table, filters_param: str | None) -> pa.Table:
+    if not filters_param:
+        return table
+    try:
+        predicates = json.loads(filters_param)
+    except (json.JSONDecodeError, TypeError):
+        return table
+    if not isinstance(predicates, list):
+        return table
+
+    mask: pa.ChunkedArray | None = None
+
+    for pred in predicates:
+        if not isinstance(pred, dict):
+            continue
+        col = pred.get("col")
+        op = pred.get("op")
+        if col not in table.schema.names or op is None:
+            continue
+
+        column = table.column(col)
+
+        if op == "is_null":
+            m = pc.is_null(column)
+        elif op == "is_not_null":
+            m = pc.is_valid(column)
+        elif op == "in":
+            value = pred.get("value")
+            if not isinstance(value, list):
+                continue
+            m = pc.is_in(column, pa.array(value))
+        elif op in _OPS:
+            value = pred.get("value")
+            if value is None:
+                continue
+            m = _OPS[op](column, pa.scalar(value))
+        else:
+            continue
+
+        mask = m if mask is None else pc.and_(mask, m)
+
+    if mask is None:
+        return table
+    return table.filter(mask)

n6k-0.1.0/src/n6k_server/routes/tables.py

@@ -0,0 +1,47 @@
+import pandas as pd
+import pyarrow as pa
+from fastapi import APIRouter, Request
+from starlette.responses import Response
+
+from n6k_server.formats import arrow_response, negotiate
+from n6k_server.pushdown import apply_columns, apply_filters
+
+router = APIRouter()
+
+
+@router.get("/")
+async def list_tables(request: Request) -> Response:
+    names = list(request.app.state.data_sources.keys())
+    table = pa.table({
+        "name": pa.array(names, type=pa.utf8()),
+        "path": pa.array([f"/tables/{n}" for n in names], type=pa.utf8()),
+        "schema_path": pa.array([f"/tables/{n}/_schema" for n in names], type=pa.utf8()),
+    })
+    return arrow_response(table, negotiate(request))
+
+
+@router.get("/{name}")
+async def get_table(name: str, request: Request, columns: str | None = None, filters: str | None = None) -> Response:
+    fn = request.app.state.data_sources.get(name)
+    if fn is None:
+        return Response(content=f"table not found: {name}", status_code=404)
+    df = fn()
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(f"data source '{name}' must return a DataFrame, got {type(df).__name__}")
+    table = pa.Table.from_pandas(df)
+    table = apply_filters(table, filters)
+    table = apply_columns(table, columns)
+    return arrow_response(table, negotiate(request))
+
+
+@router.get("/{name}/_schema")
+async def get_schema(name: str, request: Request) -> Response:
+    fn = request.app.state.data_sources.get(name)
+    if fn is None:
+        return Response(content=f"table not found: {name}", status_code=404)
+    df = fn()
+    if not isinstance(df, pd.DataFrame):
+        raise TypeError(f"data source '{name}' must return a DataFrame, got {type(df).__name__}")
+    table = pa.Table.from_pandas(df)
+    empty = table.schema.empty_table()
+    return arrow_response(empty, negotiate(request))

n6k-0.1.0/tests/test_demo.py

@@ -0,0 +1,128 @@
+import json
+from io import BytesIO
+
+import pandas as pd
+import pyarrow as pa
+import pyarrow.ipc as ipc
+import pyarrow.parquet as pq
+import pytest
+from httpx import AsyncClient, ASGITransport
+from starlette.responses import PlainTextResponse
+
+import n6k_server
+
+
+@pytest.fixture
+def app():
+    application = n6k_server.create_app()
+
+    @n6k_server.data(application, "demo")
+    def demo():
+        return pd.DataFrame({
+            "id": [1, 2, 3],
+            "name": ["alice", "bob", "charlie"],
+            "score": [95.5, 87.3, 91.8],
+        })
+
+    @application.get("/health")
+    async def health() -> PlainTextResponse:
+        return PlainTextResponse("ok")
+
+    return application
+
+
+@pytest.fixture
+def transport(app):
+    return ASGITransport(app=app)
+
+
+@pytest.fixture
+async def client(transport):
+    async with AsyncClient(transport=transport, base_url="http://test") as c:
+        yield c
+
+
+class TestHealth:
+    async def test_returns_ok(self, client: AsyncClient):
+        resp = await client.get("/health")
+        assert resp.status_code == 200
+        assert resp.text == "ok"
+
+
+class TestDemo:
+    async def test_default_returns_ipc_stream(self, client: AsyncClient):
+        resp = await client.get("/tables/demo")
+        assert resp.status_code == 200
+        assert resp.headers["content-type"] == "application/vnd.apache.arrow.stream"
+        table = ipc.open_stream(resp.content).read_all()
+        assert table.num_rows == 3
+        assert "id" in table.column_names
+        assert "name" in table.column_names
+        assert "score" in table.column_names
+
+    async def test_accept_ipc_file(self, client: AsyncClient):
+        resp = await client.get(
+            "/tables/demo",
+            headers={"accept": "application/vnd.apache.arrow.file"},
+        )
+        assert resp.status_code == 200
+        assert resp.headers["content-type"] == "application/vnd.apache.arrow.file"
+        table = ipc.open_file(resp.content).read_all()
+        assert table.num_rows == 3
+
+    async def test_accept_parquet(self, client: AsyncClient):
+        resp = await client.get(
+            "/tables/demo",
+            headers={"accept": "application/x-parquet"},
+        )
+        assert resp.status_code == 200
+        assert resp.headers["content-type"] == "application/x-parquet"
+        table = pq.read_table(BytesIO(resp.content))
+        assert table.num_rows == 3
+
+    async def test_unknown_accept_defaults_to_stream(self, client: AsyncClient):
+        resp = await client.get(
+            "/tables/demo",
+            headers={"accept": "application/json"},
+        )
+        assert resp.status_code == 200
+        assert resp.headers["content-type"] == "application/vnd.apache.arrow.stream"
+
+    async def test_roundtrip_data_integrity(self, client: AsyncClient):
+        resp = await client.get("/tables/demo")
+        table = ipc.open_stream(resp.content).read_all()
+        assert table.column("id").to_pylist() == [1, 2, 3]
+        assert table.column("name").to_pylist() == ["alice", "bob", "charlie"]
+        assert table.column("score").to_pylist() == [95.5, 87.3, 91.8]
+
+
+class TestTablesPushdown:
+    async def test_columns_projection(self, client: AsyncClient):
+        resp = await client.get("/tables/demo?columns=id,name")
+        assert resp.status_code == 200
+        table = ipc.open_stream(resp.content).read_all()
+        assert table.column_names == ["id", "name"]
+        assert table.num_rows == 3
+
+    async def test_filter_equal(self, client: AsyncClient):
+        filters = json.dumps([{"col": "id", "op": "=", "value": 1}])
+        resp = await client.get(f"/tables/demo?filters={filters}")
+        assert resp.status_code == 200
+        table = ipc.open_stream(resp.content).read_all()
+        assert table.num_rows == 1
+        assert table.column("name").to_pylist() == ["alice"]
+
+    async def test_filter_and_columns(self, client: AsyncClient):
+        filters = json.dumps([{"col": "score", "op": ">", "value": 90.0}])
+        resp = await client.get(f"/tables/demo?columns=id,name&filters={filters}")
+        assert resp.status_code == 200
+        table = ipc.open_stream(resp.content).read_all()
+        assert table.column_names == ["id", "name"]
+        assert table.num_rows == 2
+
+
+class TestExceptionHandler:
+    async def test_404_returns_plain_text(self, client: AsyncClient):
+        resp = await client.get("/nonexistent")
+        assert resp.status_code == 404
+        assert "application/json" not in resp.headers.get("content-type", "")

n6k-0.1.0/tests/test_formats.py

@@ -0,0 +1,84 @@
+import pyarrow as pa
+import pyarrow.ipc as ipc
+import pyarrow.parquet as pq
+from io import BytesIO
+from starlette.testclient import TestClient
+
+from n6k_server.formats import ArrowFormat, negotiate, arrow_response, _serialize_ipc_stream, _serialize_ipc_file, _serialize_parquet
+
+
+def _make_table() -> pa.Table:
+    return pa.table({"x": [1, 2, 3], "y": ["a", "b", "c"]})
+
+
+class _FakeRequest:
+    def __init__(self, accept: str = ""):
+        self.headers = {"accept": accept}
+
+
+class TestNegotiate:
+    def test_ipc_stream(self):
+        req = _FakeRequest("application/vnd.apache.arrow.stream")
+        assert negotiate(req) == ArrowFormat.IPC_STREAM
+
+    def test_ipc_file(self):
+        req = _FakeRequest("application/vnd.apache.arrow.file")
+        assert negotiate(req) == ArrowFormat.IPC_FILE
+
+    def test_parquet(self):
+        req = _FakeRequest("application/x-parquet")
+        assert negotiate(req) == ArrowFormat.PARQUET
+
+    def test_default_when_missing(self):
+        req = _FakeRequest("")
+        assert negotiate(req) == ArrowFormat.IPC_STREAM
+
+    def test_default_when_unknown(self):
+        req = _FakeRequest("application/json")
+        assert negotiate(req) == ArrowFormat.IPC_STREAM
+
+    def test_first_match_wins(self):
+        req = _FakeRequest("application/x-parquet, application/vnd.apache.arrow.file")
+        assert negotiate(req) == ArrowFormat.PARQUET
+
+    def test_quality_params_ignored_gracefully(self):
+        req = _FakeRequest("application/vnd.apache.arrow.file;q=0.9")
+        assert negotiate(req) == ArrowFormat.IPC_FILE
+
+
+class TestSerializers:
+    def test_ipc_stream_roundtrip(self):
+        table = _make_table()
+        data = _serialize_ipc_stream(table)
+        result = ipc.open_stream(data).read_all()
+        assert result.equals(table)
+
+    def test_ipc_file_roundtrip(self):
+        table = _make_table()
+        data = _serialize_ipc_file(table)
+        result = ipc.open_file(data).read_all()
+        assert result.equals(table)
+
+    def test_parquet_roundtrip(self):
+        table = _make_table()
+        data = _serialize_parquet(table)
+        result = pq.read_table(BytesIO(data))
+        assert result.equals(table)
+
+
+class TestArrowResponse:
+    def test_content_type_ipc_stream(self):
+        table = _make_table()
+        resp = arrow_response(table, ArrowFormat.IPC_STREAM)
+        assert resp.media_type == "application/vnd.apache.arrow.stream"
+
+    def test_content_type_parquet(self):
+        table = _make_table()
+        resp = arrow_response(table, ArrowFormat.PARQUET)
+        assert resp.media_type == "application/x-parquet"
+
+    def test_body_is_deserializable(self):
+        table = _make_table()
+        resp = arrow_response(table, ArrowFormat.IPC_STREAM)
+        result = ipc.open_stream(resp.body).read_all()
+        assert result.equals(table)