ob-flight-extension 0.1.0__tar.gz

ob_flight_extension-0.1.0/.gitignore

@@ -0,0 +1,35 @@
+__pycache__/
+*.py[cod]
+*$py.class
+*.so
+dist/
+build/
+*.egg-info/
+*.egg
+.eggs/
+.venv/
+venv/
+.env
+*.db
+*.duckdb
+*.sqlite3
+.mypy_cache/
+.ruff_cache/
+.pytest_cache/
+htmlcov/
+.coverage
+.coverage.*
+*.log
+.DS_Store
+.claude/
+security/
+infra/
+site/
+scripts/
+.mcpregistry_github_token
+.mcpregistry_registry_token
+_*/
+blog/
+review*
+design/
+examples/test*

ob_flight_extension-0.1.0/CLAUDE.md

@@ -0,0 +1,285 @@
+# ob-flight-extension — Arrow Flight SQL Server for orionbelt-api
+
+## Purpose
+
+Adds Arrow Flight SQL protocol to the existing orionbelt-api FastAPI server.
+Runs as a background thread in the same Python process — no extra container,
+no HTTP hop between Flight and the OB compilation engine.
+
+Target deployment: on-premise Docker, LAN access, enterprise data teams.
+Client tools: DBeaver, Tableau (via Flight JDBC .jar), Power BI (via ODBC bridge).
+
+---
+
+## Integration into orionbelt-api
+
+### 1. Install as optional dependency
+
+In orionbelt-api's pyproject.toml add:
+```toml
+[project.optional-dependencies]
+flight = ["ob-flight-extension>=0.1", "pyarrow>=16.0"]
+```
+
+### 2. Hook into lifespan
+
+In orionbelt/api/app.py, modify create_app():
+
+```python
+from contextlib import asynccontextmanager
+import os
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    if os.getenv("FLIGHT_ENABLED", "false").lower() == "true":
+        from ob_flight.startup import start_flight_background
+        flight_thread = start_flight_background()
+    yield
+    if os.getenv("FLIGHT_ENABLED", "false").lower() == "true":
+        from ob_flight.startup import stop_flight_server
+        stop_flight_server()
+
+app = FastAPI(lifespan=lifespan)
+```
+
+### 3. Docker port
+
+Add port 8815 to docker-compose.yml (see ARCHITECTURE.md).
+
+---
+
+## Module Map
+
+| File | Responsibility |
+|---|---|
+| startup.py | start_flight_background() — launches server in daemon thread |
+| server.py | OBFlightServer(FlightServerBase) — pyarrow server subclass |
+| handlers.py | GetFlightInfo, DoGet, GetSchema, ListFlights implementations |
+| catalog.py | Model introspection → Flight catalog (tables = data objects) |
+| auth.py | ServerAuthHandler — token or no-auth modes |
+| converters.py | DB result rows → pyarrow RecordBatch |
+| db_router.py | Routes to correct native connector based on model dialect |
+
+---
+
+## Key Implementation: GetFlightInfo
+
+This is the entry point. DBeaver calls this first with the query text.
+
+```python
+def get_flight_info(self, context, descriptor):
+    query_bytes = descriptor.command
+    query_str = query_bytes.decode("utf-8")
+
+    # Resolve model from descriptor path (= DBeaver "database" field)
+    model_id = self._resolve_model(descriptor.path)
+    model = self._model_cache[model_id]
+
+    # Compile if OBML, pass through if plain SQL
+    if is_obml(query_str):
+        obml = yaml.safe_load(query_str)
+        pipeline = CompilationPipeline()
+        result = pipeline.compile(obml, model, dialect=model.dialect)
+        sql = result.sql
+    else:
+        sql = query_str
+
+    # Store sql+model in a short-lived ticket (UUID key → in-memory dict)
+    ticket_id = str(uuid.uuid4())
+    self._pending[ticket_id] = (sql, model_id)
+    ticket = pa.flight.Ticket(ticket_id.encode())
+
+    # Return schema + ticket (schema requires a dry-run or model metadata)
+    schema = self._infer_schema(sql, model_id)
+    endpoint = pa.flight.FlightEndpoint(ticket, [])
+    return pa.flight.FlightInfo(schema, descriptor, [endpoint], -1, -1)
+```
+
+---
+
+## Key Implementation: DoGet
+
+DBeaver calls this after GetFlightInfo to stream the actual data.
+
+```python
+def do_get(self, context, ticket):
+    ticket_id = ticket.ticket.decode("utf-8")
+    sql, model_id = self._pending.pop(ticket_id)
+    model = self._model_cache[model_id]
+
+    # Execute on native connector
+    conn = db_router.connect(model)
+    cursor = conn.cursor()
+    cursor.execute(sql)
+
+    # Stream as Arrow record batches
+    schema = pa.Schema.from_pandas(...)  # or from cursor.description
+    def record_batch_generator():
+        while True:
+            rows = cursor.fetchmany(1000)
+            if not rows:
+                break
+            yield pa.RecordBatch.from_pydict(
+                {col: [r[i] for r in rows]
+                 for i, col in enumerate(col_names)},
+                schema=schema,
+            )
+    return pa.flight.RecordBatchStream(
+        pa.RecordBatchReader.from_batches(schema, record_batch_generator())
+    )
+```
+
+---
+
+## Key Implementation: ListFlights (DBeaver schema browser)
+
+DBeaver calls ListFlights to populate the schema tree on the left panel.
+Map OB model data objects → Flight "tables":
+
+```python
+def list_flights(self, context, criteria):
+    for model_id, model in self._model_cache.items():
+        for obj_name, obj in model.data_objects.items():
+            descriptor = pa.flight.FlightDescriptor.for_path(
+                model_id, obj_name
+            )
+            schema = self._schema_for_object(obj)
+            yield pa.flight.FlightInfo(
+                schema, descriptor, [], -1, -1
+            )
+```
+
+This makes DBeaver show the OB model's data objects as browsable tables,
+with correct column types visible before any query is run.
+
+---
+
+## db_router.py — Vendor Routing
+
+Routes execution to the correct native connector based on the model's
+declared dialect or an explicit env override:
+
+```python
+VENDOR_MAP = {
+    "snowflake":  "ob_snowflake",
+    "postgres":   "ob_postgres",
+    "clickhouse": "ob_clickhouse",
+    "dremio":     "ob_dremio",
+    "databricks": "ob_databricks",
+}
+
+def connect(model: SemanticModel) -> Connection:
+    dialect = model.dialect or os.getenv("DB_VENDOR", "snowflake")
+    module = importlib.import_module(VENDOR_MAP[dialect])
+    return module.connect(**_creds_from_env(dialect))
+```
+
+Credentials come from environment variables per vendor:
+  SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PASSWORD, ...
+  POSTGRES_HOST, POSTGRES_PORT, POSTGRES_USER, ...
+  etc.
+
+---
+
+## Startup Thread
+
+```python
+import threading
+import pyarrow.flight as flight
+
+_server: flight.FlightServerBase | None = None
+
+def start_flight_background() -> threading.Thread:
+    global _server
+    port = int(os.getenv("FLIGHT_PORT", "8815"))
+    _server = OBFlightServer(location=f"grpc://0.0.0.0:{port}")
+    _server.init()   # load models, init cache
+    thread = threading.Thread(
+        target=_server.serve,
+        name="ob-flight-server",
+        daemon=True,   # dies with the main process
+    )
+    thread.start()
+    return thread
+
+def stop_flight_server() -> None:
+    if _server:
+        _server.shutdown()
+```
+
+daemon=True is critical — ensures the Flight server thread dies cleanly
+when the FastAPI process exits or is killed by Docker.
+
+---
+
+## Environment Variables
+
+| Variable | Default | Description |
+|---|---|---|
+| FLIGHT_ENABLED | false | Set to "true" to activate |
+| FLIGHT_PORT | 8815 | gRPC listen port |
+| FLIGHT_AUTH_MODE | none | "none" or "token" |
+| FLIGHT_API_TOKEN | | Static token for auth mode "token" |
+| FLIGHT_DEFAULT_MODEL | | Fallback model_id if path empty |
+| FLIGHT_PRELOAD_MODELS | | Comma-sep .obml.yaml paths, loaded at startup |
+| DB_VENDOR | snowflake | Default vendor for db_router |
+
+---
+
+## DBeaver Configuration (end-user guide)
+
+1. New Connection → Apache Arrow Flight SQL
+2. Host: <docker-host-ip>  Port: 8815
+3. Database: <model_name_or_id>   (e.g. "orionbelt_1")
+4. Authentication: leave empty (FLIGHT_AUTH_MODE=none on LAN)
+5. Test Connection → should show "Connected"
+6. In SQL editor, write either:
+   - Plain SQL:  SELECT * FROM ORDERS LIMIT 10
+   - OBML YAML:
+       select:
+         dimensions: [Region]
+         measures: [Revenue]
+7. Schema browser shows OB model data objects as tables
+
+---
+
+## Tableau Configuration (end-user guide)
+
+1. Download flight-sql-jdbc-driver-*.jar from Apache Arrow releases
+2. Tableau → Connect → Other Databases (JDBC)
+3. URL: jdbc:arrow-flight-sql://<host>:8815?useEncryption=false
+4. Username/Password: leave empty or use token
+5. Custom SQL uses same OBML YAML or plain SQL
+
+---
+
+## Build Order
+
+Session 1: converters.py + tests (rows → Arrow RecordBatch, no Flight needed)
+Session 2: server.py + startup.py skeleton (pyarrow server, no handlers yet)
+Session 3: handlers.py GetFlightInfo + DoGet (core query path)
+Session 4: catalog.py + ListFlights (DBeaver schema browser)
+Session 5: auth.py + db_router.py + integration test with DBeaver
+
+---
+
+## Dependencies
+
+```toml
+[project.dependencies]
+pyarrow = ">=16.0"
+pyyaml = ">=6.0"
+
+[project.optional-dependencies]
+snowflake = ["snowflake-connector-python>=3.0"]
+postgres  = ["adbc-driver-postgresql>=1.0"]
+clickhouse = ["clickhouse-connect>=0.7"]
+dremio    = ["pyarrow>=16.0"]   # Dremio uses Arrow Flight natively
+databricks = ["databricks-sql-connector>=3.0"]
+all = [
+    "snowflake-connector-python>=3.0",
+    "adbc-driver-postgresql>=1.0",
+    "clickhouse-connect>=0.7",
+    "databricks-sql-connector>=3.0",
+]
+```

ob_flight_extension-0.1.0/LICENSE

@@ -0,0 +1,70 @@
+Business Source License 1.1
+
+Parameters
+
+Licensor:             RALFORION d.o.o.
+Licensed Work:        OrionBelt Semantic Layer 1.0
+                      The Licensed Work is (c) 2025 RALFORION d.o.o.
+Additional Use Grant: You may use the Licensed Work for any purpose,
+                      including production use, PROVIDED THAT you may
+                      not embed OrionBelt in a commercial product or
+                      provide it as part of a commercial service
+                      without purchasing a commercial license from
+                      the Licensor. Contact licensing@ralforion.com
+                      for commercial licensing inquiries.
+Change Date:          2030-03-16
+Change License:       Apache License, Version 2.0
+
+-----------------------------------------------------------------------------
+
+Business Source License 1.1
+
+License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved.
+"Business Source License" is a trademark of MariaDB Corporation Ab.
+
+Terms
+
+The Licensor hereby grants you the right to copy, modify, create derivative
+works, redistribute, and make non-production use of the Licensed Work. The
+Licensor may make an Additional Use Grant, above, permitting limited
+production use.
+
+Effective on the Change Date, or the fourth anniversary of the first publicly
+available distribution of a specific version of the Licensed Work under this
+License, whichever comes first, the Licensor hereby grants you rights under
+the terms of the Change License, and the rights granted in the paragraph
+above terminate.
+
+If your use of the Licensed Work does not comply with the requirements
+currently in effect as described in this License, you must purchase a
+commercial license from the Licensor, its affiliated entities, or authorized
+resellers, or you must refrain from using the Licensed Work.
+
+All copies of the original and modified Licensed Work, and derivative works
+of the Licensed Work, are subject to this License. This License applies
+separately for each version of the Licensed Work and the Change Date may vary
+for each version of the Licensed Work released by Licensor.
+
+You must conspicuously display this License on each original or modified copy
+of the Licensed Work. If you receive the Licensed Work in original or
+modified form from a third party, the terms and conditions set forth in this
+License apply to your use of that work.
+
+Any use of the Licensed Work in violation of this License will automatically
+terminate your rights under this License for the current and all other
+versions of the Licensed Work.
+
+This License does not grant you any right in any trademark or logo of
+Licensor or its affiliates (provided that you may use a trademark or logo of
+Licensor as expressly required by this License).
+
+TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON
+AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS,
+EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND
+TITLE.
+
+MariaDB hereby grants you permission to use this License's text to license
+your works, and to refer to it using the trademark "Business Source License",
+as long as you comply with the Covenants of Licensor of MariaDB's Business Source
+License Trademark Policy available at https://mariadb.com/bsl11/.

ob_flight_extension-0.1.0/PKG-INFO

@@ -0,0 +1,53 @@
+Metadata-Version: 2.4
+Name: ob-flight-extension
+Version: 0.1.0
+Summary: Arrow Flight SQL server extension for orionbelt-api (DBeaver, Tableau, Power BI)
+Author-email: Ralf Becher <info@orionbelt.ai>
+License: BSL-1.1
+License-File: LICENSE
+Keywords: arrow,dbeaver,flight-sql,orionbelt,semantic-layer,tableau
+Requires-Python: >=3.12
+Requires-Dist: ob-driver-core>=0.1
+Requires-Dist: pyarrow>=16.0
+Requires-Dist: pyyaml>=6.0
+Provides-Extra: all
+Requires-Dist: ob-clickhouse>=0.1; extra == 'all'
+Requires-Dist: ob-databricks>=0.1; extra == 'all'
+Requires-Dist: ob-dremio>=0.1; extra == 'all'
+Requires-Dist: ob-duckdb>=0.1; extra == 'all'
+Requires-Dist: ob-postgres>=0.1; extra == 'all'
+Requires-Dist: ob-snowflake>=0.1; extra == 'all'
+Provides-Extra: clickhouse
+Requires-Dist: ob-clickhouse>=0.1; extra == 'clickhouse'
+Provides-Extra: databricks
+Requires-Dist: ob-databricks>=0.1; extra == 'databricks'
+Provides-Extra: dev
+Requires-Dist: mypy>=1.10; extra == 'dev'
+Requires-Dist: pytest-asyncio>=0.23; extra == 'dev'
+Requires-Dist: pytest>=8.0; extra == 'dev'
+Requires-Dist: ruff>=0.4; extra == 'dev'
+Provides-Extra: dremio
+Requires-Dist: ob-dremio>=0.1; extra == 'dremio'
+Provides-Extra: duckdb
+Requires-Dist: ob-duckdb>=0.1; extra == 'duckdb'
+Provides-Extra: postgres
+Requires-Dist: ob-postgres>=0.1; extra == 'postgres'
+Provides-Extra: snowflake
+Requires-Dist: ob-snowflake>=0.1; extra == 'snowflake'
+Description-Content-Type: text/markdown
+
+# ob-flight-extension
+
+Arrow Flight SQL server extension for orionbelt-api (DBeaver, Tableau, Power BI)
+
+Part of the [OrionBelt Semantic Layer](https://github.com/ralfbecher/orionbelt-semantic-layer) project.
+
+## Installation
+
+```bash
+pip install ob-flight-extension
+```
+
+## License
+
+BSL-1.1 — see [LICENSE](LICENSE) for details.

ob_flight_extension-0.1.0/README.md

@@ -0,0 +1,15 @@
+# ob-flight-extension
+
+Arrow Flight SQL server extension for orionbelt-api (DBeaver, Tableau, Power BI)
+
+Part of the [OrionBelt Semantic Layer](https://github.com/ralfbecher/orionbelt-semantic-layer) project.
+
+## Installation
+
+```bash
+pip install ob-flight-extension
+```
+
+## License
+
+BSL-1.1 — see [LICENSE](LICENSE) for details.

ob_flight_extension-0.1.0/docker-compose.example.yml

@@ -0,0 +1,39 @@
+# docker-compose.yml — add to your existing orionbelt-api service
+# Only two lines change: expose port 8815 and add FLIGHT_ENABLED=true
+
+services:
+  orionbelt:
+    image: orionbelt-api:latest
+    build: .
+    ports:
+      - "8000:8000"    # REST API (existing)
+      - "8815:8815"    # Arrow Flight SQL (NEW)
+    environment:
+      # --- existing env vars unchanged ---
+      LOG_LEVEL: INFO
+      API_SERVER_HOST: "0.0.0.0"
+      API_SERVER_PORT: "8000"
+      SESSION_TTL_SECONDS: "1800"
+
+      # --- Arrow Flight SQL (NEW) ---
+      FLIGHT_ENABLED: "true"
+      FLIGHT_PORT: "8815"
+      FLIGHT_AUTH_MODE: "none"          # use "token" for production
+      # FLIGHT_API_TOKEN: "changeme"    # required when AUTH_MODE=token
+      FLIGHT_DEFAULT_MODEL: ""          # fallback if DBeaver sends no path
+      FLIGHT_PRELOAD_MODELS: ""         # comma-sep paths, loaded at startup
+
+      # --- DB credentials (used by db_router.py) ---
+      DB_VENDOR: snowflake              # default vendor
+      SNOWFLAKE_ACCOUNT: "xy12345.eu-west-1"
+      SNOWFLAKE_USER: "your_user"
+      SNOWFLAKE_PASSWORD: "your_password"
+      SNOWFLAKE_DATABASE: "MY_DB"
+      SNOWFLAKE_SCHEMA: "PUBLIC"
+      SNOWFLAKE_WAREHOUSE: "COMPUTE_WH"
+      # POSTGRES_HOST: localhost
+      # POSTGRES_PORT: 5432
+      # POSTGRES_USER: postgres
+      # POSTGRES_PASSWORD: secret
+      # POSTGRES_DBNAME: mydb
+    restart: unless-stopped

ob_flight_extension-0.1.0/pyproject.toml

@@ -0,0 +1,58 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "ob-flight-extension"
+version = "0.1.0"
+description = "Arrow Flight SQL server extension for orionbelt-api (DBeaver, Tableau, Power BI)"
+license = { text = "BSL-1.1" }
+readme = "README.md"
+authors = [{ name = "Ralf Becher", email = "info@orionbelt.ai" }]
+keywords = ["arrow", "flight-sql", "dbeaver", "tableau", "orionbelt", "semantic-layer"]
+requires-python = ">=3.12"
+dependencies = [
+    "pyarrow>=16.0",
+    "pyyaml>=6.0",
+    "ob-driver-core>=0.1",
+]
+
+[project.optional-dependencies]
+duckdb      = ["ob-duckdb>=0.1"]
+snowflake   = ["ob-snowflake>=0.1"]
+postgres    = ["ob-postgres>=0.1"]
+clickhouse  = ["ob-clickhouse>=0.1"]
+dremio      = ["ob-dremio>=0.1"]
+databricks  = ["ob-databricks>=0.1"]
+all = [
+    "ob-duckdb>=0.1",
+    "ob-snowflake>=0.1",
+    "ob-postgres>=0.1",
+    "ob-clickhouse>=0.1",
+    "ob-dremio>=0.1",
+    "ob-databricks>=0.1",
+]
+dev = [
+    "pytest>=8.0",
+    "pytest-asyncio>=0.23",
+    "mypy>=1.10",
+    "ruff>=0.4",
+]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/ob_flight"]
+
+[tool.ruff]
+line-length = 100
+target-version = "py312"
+
+[tool.mypy]
+python_version = "3.12"
+strict = true
+files = ["src/ob_flight"]
+
+[tool.pytest.ini_options]
+asyncio_mode = "auto"
+testpaths = ["tests"]
+markers = ["integration: requires running orionbelt-api + DBeaver"]
+addopts = "-m 'not integration'"

ob_flight_extension-0.1.0/src/ob_flight/__init__.py

@@ -0,0 +1,36 @@
+"""ob-flight-extension — Arrow Flight SQL server for OrionBelt Semantic Layer.
+
+Adds Arrow Flight SQL protocol to the orionbelt-api FastAPI server.
+Runs as a daemon thread in the same process — no HTTP hop for compilation.
+
+Usage: Set FLIGHT_ENABLED=true when starting orionbelt-api.
+
+Note: server and startup are NOT imported at package level to avoid
+pulling in pyarrow.flight (heavy gRPC init) when only db_router is needed.
+Import them explicitly: ``from ob_flight.startup import start_flight_background``
+"""
+
+from __future__ import annotations
+
+__all__ = [
+    "OBFlightServer",
+    "start_flight_background",
+    "stop_flight_server",
+]
+
+
+def __getattr__(name: str) -> object:
+    """Lazy imports for server and startup — avoids pyarrow.flight at import time."""
+    if name == "OBFlightServer":
+        from ob_flight.server import OBFlightServer
+
+        return OBFlightServer
+    if name == "start_flight_background":
+        from ob_flight.startup import start_flight_background
+
+        return start_flight_background
+    if name == "stop_flight_server":
+        from ob_flight.startup import stop_flight_server
+
+        return stop_flight_server
+    raise AttributeError(f"module 'ob_flight' has no attribute {name!r}")

ob_flight_extension-0.1.0/src/ob_flight/auth.py

@@ -0,0 +1,55 @@
+"""Authentication handlers for the Flight SQL server."""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+import pyarrow.flight as flight
+
+
+class NoopAuthHandler(flight.ServerAuthHandler):
+    """No authentication — accept all connections."""
+
+    def authenticate(self, outgoing: Any, incoming: Any) -> None:
+        """Accept all connections without authentication."""
+
+    def is_valid(self, token: bytes) -> str:
+        """All tokens are valid — return empty peer identity."""
+        return ""
+
+
+class TokenAuthHandler(flight.ServerAuthHandler):
+    """Simple static bearer token authentication."""
+
+    def __init__(self, token: str) -> None:
+        super().__init__()
+        self._token = token.encode("utf-8")
+
+    def authenticate(self, outgoing: Any, incoming: Any) -> None:
+        """Validate the token from the client."""
+        buf = incoming.read()
+        if buf != self._token:
+            raise flight.FlightUnauthenticatedError("Invalid token")
+        outgoing.write(self._token)
+
+    def is_valid(self, token: bytes) -> str:
+        """Check if the token matches."""
+        if token != self._token:
+            raise flight.FlightUnauthenticatedError("Invalid token")
+        return "authenticated"
+
+
+def create_auth_handler() -> flight.ServerAuthHandler:
+    """Create an auth handler based on environment variables.
+
+    FLIGHT_AUTH_MODE=none (default) -> NoopAuthHandler
+    FLIGHT_AUTH_MODE=token -> TokenAuthHandler(FLIGHT_API_TOKEN)
+    """
+    mode = os.getenv("FLIGHT_AUTH_MODE", "none").lower()
+    if mode == "token":
+        token = os.getenv("FLIGHT_API_TOKEN", "")
+        if not token:
+            raise ValueError("FLIGHT_AUTH_MODE=token requires FLIGHT_API_TOKEN to be set")
+        return TokenAuthHandler(token)
+    return NoopAuthHandler()